|
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544 |
- // #region web speech recognition api
- var SpeechRecognition = SpeechRecognition || webkitSpeechRecognition;
- var SpeechGrammarList = SpeechGrammarList || webkitSpeechGrammarList;
- var SpeechRecognitionEvent = SpeechRecognitionEvent || webkitSpeechRecognitionEvent;
- // #endregion
-
- // #region state management
- var state = '';
- var question = 0;
- var rePrompt = false;
- var partTwo = false;
- var questionThreeCount = 0;
- var strike = 0;
- // #endregion
-
- // #region questions
- const QUESTION_ONE = 'Ich werde Ihnen jetzt langsam eine Liste mit Worten vorlesen. Danach wiederholen Sie bitte möglichst viele dieser Worte. Auf die Reihenfolge kommt es nicht an.';
- const QUESTION_ONE_PT2 = 'Vielen Dank. Nun nenne ich Ihnen die gleichen 10 Worte ein zweites mal. Auch danach sollen Sie wieder möglichst viele Worte wiederholen';
- const QUESTION_TWO = 'Nennen Sie mir bitte so viel Dinge wie möglich, die man im Supermarkt kaufen kann. Sie haben dafür eine Minute Zeit. Und Los';
- const QUESTION_THREE = 'Ich werde Ihnen jetzt eine Zahlenreihe nennen, die Sie mir dann bitte in umgekehrter Reihenfolge wiederholen sollen. Wenn ich beispielsweise, vier - fünf sage, dann sagen Sie bitte, fünf - vier.';
- // #endregion
-
- // #region intents
- const WELCOME_INTENT = 'Default Welcome Intent';
- const WELCOME_FOLLOWUP_YES = 'Default Welcome Intent - yes';
- const WELCOME_FOLLOWUP_NO = 'Default Welcome Intent - no';
- const MORE_TIME = 'Add Time Intent';
- const MORE_TIME_YES = 'Add Time Intent - yes';
- const MORE_TIME_NO = 'Add Time Intent - no';
- const QUIT_INTENT = 'Quit Intent';
- const FALLBACK_INTENT = 'Default Fallback Intent';
- const HELP_INTENT = 'Help Intent';
- const CHOOSE_QUESTION = 'Frage_Starten';
- const NEXT_QUESTION = 'Nächste Frage';
- // #endregion
-
- // #region questions and expected results
- const QUESTION_ONE_ANSWERS = { 'teller': 1, 'hund': 1, 'lampe': 1, 'brief': 1, 'apfel': 1, 'apfelwiese': 2, 'apfelbaum': 2, 'und': 1, 'hose': 1, 'tisch': 1, 'wiese': 1, 'glas': 1, 'baum': 1 };
- const QUESTION_ONE_QUESTIONS = ['teller', 'hund', 'lampe', 'brief', 'apfel', 'hose', 'tisch', 'wiese', 'glas', 'baum'];
- const QUESTION_TWO_ANSWERS = {};
- var QUESTION_TWO_QUESTIONS = [''];
- const QUESTION_THREE_QUESTIONS_PT1 = ['7, 2', '4, 7, 9', '5, 4, 9, 6', '2, 7, 5, 3, 6', '8, 1, 3, 5, 4, 2'];
- const QUESTION_THREE_QUESTIONS_PT2 = ['8, 6', '3, 1, 5', '1, 9, 7, 4', '1, 3, 5, 4, 8', '4, 1, 2, 7, 9, 5'];
- const QUESTION_THREE_ANSWERS_PT1 = ['27', '974', '6945', '63572', '245318'];
- const QUESTION_THREE_ANSWERS_PT2 = ['68', '513', '4791', '84531', '597214'];
- LoadQuestionTwo();
- function LoadQuestionTwo () {
- var xmlhttp;
- if (window.XMLHttpRequest) { // code for IE7+, Firefox, Chrome, Opera, Safari
- xmlhttp = new XMLHttpRequest();
- } else { // code for IE6, IE5
- xmlhttp = new ActiveXObject('Microsoft.XMLHTTP');
- }
- xmlhttp.onreadystatechange = function () {
- if (xmlhttp.readyState === 4 && xmlhttp.status === 200) {
- var text = xmlhttp.responseText.toLowerCase();
- // Now convert it into array using regex
- QUESTION_TWO_QUESTIONS = text.split('\r\n');
- for (let word of QUESTION_TWO_QUESTIONS) {
- QUESTION_TWO_ANSWERS[word] = 1;
- }
- }
- };
- xmlhttp.open('GET', 'lebensmittel.txt', true);
- xmlhttp.send();
- }
- // #endregion
-
- // #region points
- var questionOnePoints = 0;
- var questionTwoPoints = 0;
- var questionThreePoints = 0;
- var questionFourPoints = 0;
- var questionFivePoints = 0;
- var questionSixPoints = 0;
- // #endregion
-
- // tokenization
- const separators = [' ', '\\\+', '-', '\\\(', '\\\)', '\\*', '/', ':', '\\\?'];
-
- // Timers
- var timerId;
-
- // #region html elements
- var serverPara = document.querySelector('.server');
- var diagnosticPara = document.querySelector('.output');
- var testBtn = document.querySelector('button');
- var testBtn2 = document.getElementById('speechBtn');
- var infoPara = document.getElementById('info');
- var userPrompt = document.getElementById('query');
- // #endregion
-
- // websocket to communicate with the server
- var ws = new WebSocket('ws://' + window.location.host + window.location.pathname + 'ws');
-
- // #region speech recognition initialization
- var recognition = new SpeechRecognition();
- recognition.lang = 'de-DE';
- // recognition.interimResults = false;
- recognition.maxAlternatives = 1;
- recognition.continuous = true;
- var answerQuery = '';
- var skipRecording = false;
- // #endregion
-
- // #region speech synthesis initialization
- var speechsynth = new SpeechSynthesisUtterance();
- var listSpeechsynth = new SpeechSynthesisUtterance();
- var voices;
- // #endregion
-
- // #region speech events
- window.speechSynthesis.onvoiceschanged = function () {
- voices = window.speechSynthesis.getVoices();
- voices.forEach(element => {
- if (element.name === 'Google Deutsch') {
- speechsynth.voice = element;
- listSpeechsynth.voice = element;
- }
- });
-
- listSpeechsynth.rate = 0.7;
- };
-
- speechsynth.onend = function (event) {
- switch (question) {
- case 1:
- break;
- case 2:
- break;
- case 3:
- break;
- case 4:
- break;
- case 5:
- break;
- }
- if (!skipRecording) {
- recognizeSpeech();
- }
- skipRecording = false;
- diagnosticPara = '';
- console.log('global speech end');
- };
- // #endregion
-
- // #region websocket events
- ws.onopen = function () {
- serverPara.style.background = 'green';
- serverPara.innerHTML = 'Server online';
- };
-
- ws.onmessage = function (payload) {
- var dialogflowResult = JSON.parse(payload.data);
- checkIntent(dialogflowResult);
- document.querySelector('h1').innerHTML = dialogflowResult.intent.displayName;
- };
- // #endregion
-
- // INTENT HANDLING
- function checkIntent (result) {
- switch (result.intent.displayName) {
- case QUIT_INTENT:
- state = 'quit';
- if (timerId !== undefined) {
- clearTimeout(timerId);
- }
- skipRecording = true;
- speak('Beende die Durchführung.');
- break;
- case WELCOME_INTENT:
- state = 'detect';
- // speak(result.fulfillmentText)
- speak('go');
- break;
- case WELCOME_FOLLOWUP_YES:
- startQuestion(2);
- break;
- case WELCOME_FOLLOWUP_NO:
- skipRecording = true;
- speak('Okay, Danke fürs Benutzen.');
- break;
- case MORE_TIME:
- state = 'detect';
- speak('Brauchen Sie noch etwas Zeit?');
- break;
- case MORE_TIME_YES:
- rePrompt = true;
- state = 'answer';
- speak('Alles klar');
- break;
- case MORE_TIME_NO:
- skipRecording = true;
- state = 'answer';
- speak('Verstanden');
- recognition.stop();
- ws.send(answerQuery);
- break;
- case CHOOSE_QUESTION:
- question = result.parameters.fields.num.numberValue;
- state = 'answer';
- handleQuestion();
- break;
- case FALLBACK_INTENT:
- // if (state === 'answer') {
- // handleAnswer(result.queryText)
- // }
- break;
- default:
- break;
- }
- }
-
- // #region question handling
- function startQuestion (number) {
- question = number;
- state = 'answer';
- handleQuestion();
- }
-
- function handleQuestion () {
- switch (question) {
- case 1:
- skipRecording = true;
- speak(QUESTION_ONE);
- readQuestionOne();
- break;
- case 2:
- readQuestionTwo();
- break;
- case 3:
- readQuestionThree();
- break;
- case 4:
- break;
- case 5:
- break;
- }
- }
-
- function readQuestionOne () {
- for (let i = 0; i < QUESTION_ONE_QUESTIONS.length; i++) {
- let utterance = new SpeechSynthesisUtterance();
- utterance.voice = voices[2];
- utterance.rate = 0.75;
- utterance.text = QUESTION_ONE_QUESTIONS[i];
- window.speechSynthesis.speak(utterance);
- if (i === 9) {
- utterance.onend = function (event) {
- recognizeSpeech();
- };
- }
- }
- }
-
- function readQuestionTwo () {
- let utterance = new SpeechSynthesisUtterance();
- utterance.voice = voices[2];
- utterance.text = QUESTION_TWO;
- window.speechSynthesis.speak(utterance);
- utterance.onend = function (event) {
- window.setTimeout(
- function () {
- recognition.stop();
- handleAnswer(answerQuery);
- }, 6000);
- recognizeSpeech();
- };
- }
-
- function readQuestionThree () {
- recognition = false;
- speak('Dankeschön. Weiter geht es mit der nächsten Frage. ');
- let utterance = new SpeechSynthesisUtterance();
- utterance.voice = voices[2];
- utterance.text = QUESTION_THREE;
- window.speechSynthesis.speak(utterance);
- utterance.onend = function (event) {
- console.log('speach end');
- speak(QUESTION_THREE_QUESTIONS_PT1[questionThreeCount]);
- };
- utterance.onerror = function (event) {
- console.log('An error has occurred with the speech synthesis: ' + event.error);
- };
- }
-
- function handleAnswer (query) {
- switch (question) {
- case 1:
- handleAnswerToFirstQuestion(query);
- break;
- case 2:
- handleAnswerToSecondQuestion(query);
- break;
- case 3:
- handleAnswerToThirdQuestion(query);
- break;
- case 4:
- break;
- case 5:
- break;
- }
- }
-
- function handleAnswerToFirstQuestion (answer) {
- var tokens = answer.split(new RegExp(separators.join('|'), 'g'));
- questionOnePoints += calculatePoints(tokens, QUESTION_ONE_ANSWERS);
- if (partTwo) {
- partTwo = false;
- skipRecording = true;
- speak('Vielen Dank, nun geht es weiter mit der nächsten Frage');
- startQuestion(2);
- // state = 'detect'
- } else {
- rePrompt = false;
- skipRecording = true;
- speak(QUESTION_ONE_PT2);
- readQuestionOne(QUESTION_ONE);
- partTwo = true;
- }
- }
-
- function handleAnswerToSecondQuestion (answer) {
- var tokens = answer.split(new RegExp(separators.join('|'), 'g'));
- questionTwoPoints = calculatePoints(tokens, QUESTION_TWO_ANSWERS);
- startQuestion(3);
- // state = 'detect'
- }
-
- function handleAnswerToThirdQuestion (query) {
- speechsynth.rate = 0.87;
- query = query.replace(' ', '');
-
- let answerArray;
- let questionArray;
- if (!partTwo) {
- answerArray = QUESTION_THREE_ANSWERS_PT1;
- } else {
- answerArray = QUESTION_THREE_ANSWERS_PT2;
- }
- if (query === answerArray[questionThreeCount]) {
- strike = 0;
- partTwo = false;
- questionThreeCount++;
- questionThreePoints = questionThreeCount + 1;
- questionArray = QUESTION_THREE_QUESTIONS_PT1;
- } else {
- strike++;
- partTwo = true;
- questionArray = QUESTION_THREE_QUESTIONS_PT2;
- }
-
- if (strike === 2 || questionThreeCount === 5) {
- speechsynth.rate = 1;
- skipRecording = true;
- speak('weiter geht es mit der Nächsten Frage');
- startQuestion(4);
- return;
- }
-
- speak(questionArray[questionThreeCount]);
-
- console.log('count: ' + questionThreeCount + ', strike: ' + strike + ', points: ' + questionThreePoints);
- }
- // #endregion
-
- // #region global functions
- function startDemenzScreening () {
- // ws.send('starte demenz test');
- startQuestion(3);
- testBtn.disabled = true;
- testBtn.textContent = 'Test in progress';
- infoPara.textContent = 'wait...';
- diagnosticPara.textContent = 'detecting...';
- }
-
- function speak (sentence) {
- speechsynth.text = sentence;
- window.speechSynthesis.speak(speechsynth);
- }
-
- function testSpeechOut () {
- console.log('click');
- speechsynth.text = 'test 123';
- speechsynth.volume = 1;
- speechsynth.rate = 1;
- console.log(speechsynth);
- window.speechSynthesis.speak(speechsynth);
- console.log(window.speechSynthesis);
- }
-
- function recognizeSpeech () {
- // if (state === 'answer') {
- // var arr;
- // switch (question) {
- // case 1:
- // arr = QUESTION_ONE_QUESTIONS;
- // break;
- // case 2:
- // // arr = QUESTION_TWO_QUESTIONS;
- // break;
- // case 3:
- // arr = [1, 2, 3, 4, 5, 6, 7, 8, 9];
- // break;
- // case 4:
- // break;
- // case 5:
- // break;
- // }
-
- // // var grammar = '#JSGF V1.0; grammar colors; public <color> = ' + arr.join(' | ') + ' ;';
- // // var speechRecognitionList = new SpeechGrammarList();
- // // speechRecognitionList.addFromString(grammar, 1);
- // // recognition.grammars = speechRecognitionList;
- // }
- recognition.start();
- console.log('reocgnition started. Question: ' + question);
-
- recognition.onresult = function (event) {
- var last = event.results.length - 1;
- var speechResult = event.results[last][0].transcript.toLowerCase();
-
- diagnosticPara.textContent += speechResult + ' ';
-
- // console.log('Confidence: ' + event.results[0][0].confidence)
- console.log('process: ' + speechResult);
- processSpeech(speechResult);
- // testBtn.disabled = false
- // testBtn.textContent = 'record...'
- };
-
- function processSpeech (speechResult) {
- console.log('To dialogflow: ' + speechResult);
- ws.send(speechResult);
-
- let timeOut;
- switch (question) {
- case 1:
- timeOut = 6500;
- break;
- case 2:
- answerQuery += speechResult;
- return;
- case 3:
- if (speechResult.includes('uhr')) {
- speechResult = speechResult.replace('uhr', '');
- }
- timeOut = 6500;
- break;
- case 4:
- break;
- case 5:
- timeOut = 6500;
- break;
- }
-
- if (state === 'answer') {
- if (timerId != undefined) {
- clearTimeout(timerId);
- }
- answerQuery += speechResult;
- timerId = window.setTimeout(
- function () {
- // if (!rePrompt) {
- // ws.send('ich brauche noch etwas Zeit')
- // } else {
- console.log('recording end. Evaluate: ' + answerQuery);
- handleAnswer(answerQuery);
- answerQuery = '';
- diagnosticPara.textContent = '';
- // }
- recognition.stop();
- console.log('timer fallback');
- }, timeOut);
- } else {
- console.log('recording end.');
- recognition.stop();
- }
- }
-
- recognition.onspeechend = function () {
- // recognition.stop();
- // testBtn.disabled = false;
- // testBtn.textContent = 'Start new test';
- };
-
- recognition.onerror = function (event) {
- testBtn.disabled = false;
- testBtn.textContent = 'Start new test';
- diagnosticPara.textContent = 'Error occurred in recognition: ' + event.error;
- };
-
- recognition.onaudiostart = function (event) {
- // Fired when the user agent has started to capture audio.
-
- };
-
- recognition.onaudioend = function (event) {
-
- };
-
- recognition.onend = function (event) {
- // Fired when the speech recognition service has disconnected.
-
- };
-
- recognition.onnomatch = function (event) {
- // Fired when the speech recognition service returns a final result with no significant recognition. This may involve some degree of recognition, which doesn't meet or exceed the confidence threshold.
- // console.log('SpeechRecognition.onnomatch')
- };
-
- recognition.onsoundstart = function (event) {
- // Fired when any sound — recognisable speech or not — has been detected.
-
- };
-
- recognition.onsoundend = function (event) {
- // Fired when any sound — recognisable speech or not — has stopped being detected.
-
- };
-
- recognition.onspeechstart = function (event) {
- // Fired when sound that is recognised by the speech recognition service as speech has been detected.
-
- };
- recognition.onstart = function (event) {
- // Fired when the speech recognition service has begun listening to incoming audio with intent to recognize grammars associated with the current SpeechRecognition.
-
- };
- }
-
- function calculatePoints (tokens, dict) {
- let points = 0;
- for (let word of tokens) {
- if (dict[word] !== undefined) {
- points += dict[word];
- }
- }
- return points;
- }
- // #endregion
-
- testBtn.addEventListener('click', startDemenzScreening);
- testBtn2.addEventListener('click', testSpeechOut);
|