// #region web speech recognition api var SpeechRecognition = SpeechRecognition || webkitSpeechRecognition; var SpeechGrammarList = SpeechGrammarList || webkitSpeechGrammarList; var SpeechRecognitionEvent = SpeechRecognitionEvent || webkitSpeechRecognitionEvent; // #endregion // #region state management var state = ''; var question = 0; // currently disabled feature // var rePrompt = false; var partTwo = false; var questionThreeCount = 0; var strike = 0; // #endregion // #region questions const QUESTION_ONE = 'Ich werde Ihnen jetzt langsam eine Liste mit Worten vorlesen. Danach wiederholen Sie bitte möglichst viele dieser Worte. Auf die Reihenfolge kommt es nicht an.'; const QUESTION_ONE_PT2 = 'Vielen Dank. Nun nenne ich Ihnen die gleichen 10 Worte ein zweites mal. Auch danach sollen Sie wieder möglichst viele Worte wiederholen'; const QUESTION_TWO = 'Nennen Sie mir bitte so viel Dinge wie möglich, die man im Supermarkt kaufen kann. Sie haben dafür eine Minute Zeit. Und Los'; const QUESTION_THREE = 'Ich werde Ihnen jetzt eine Zahlenreihe nennen, die Sie mir dann bitte in umgekehrter Reihenfolge wiederholen sollen. Wenn ich beispielsweise, vier - fünf sage, dann sagen Sie bitte, fünf - vier.'; const QUESTION_FOUR = 'Wir kommen nun zur vierten Frage. Bitte zählen Sie von 76 an rückwärts. Nennen Sie die Zahlen abwechselnd als ganze Zahlen und als einzelne Ziffern. Zum Beispiel:'; const QUESTION_FOUR_PT2 = ['34', '3, 3', '32', '3, 1']; const QUESTION_FIVE = 'Ganz am Anfang dieses Tests habe ich Ihnen 10 Worte genannt. Können Sie sich noch an diese Worte erinnern?'; // #endregion const FINISH = 'Sie sind jetzt fertig mit dem Test, Vielen Dank.'; // #region intents const WELCOME_INTENT = 'Default Welcome Intent'; const WELCOME_FOLLOWUP_YES = 'Default Welcome Intent - yes'; const WELCOME_FOLLOWUP_NO = 'Default Welcome Intent - no'; // currently disabled feature // const MORE_TIME = 'Add Time Intent'; // const MORE_TIME_YES = 'Add Time Intent - yes'; // const MORE_TIME_NO = 'Add Time Intent - no'; const QUIT_INTENT = 'Quit Intent'; const FALLBACK_INTENT = 'Default Fallback Intent'; const HELP_INTENT = 'Help Intent'; const CHOOSE_QUESTION = 'Frage_Starten'; // disabled feature // const NEXT_QUESTION = 'Nächste Frage'; // #endregion // #region questions and expected results const QUESTION_ONE_ANSWERS = { 'teller': 1, 'hund': 1, 'lampe': 1, 'brief': 1, 'apfel': 1, 'apfelwiese': 2, 'apfelbaum': 2, 'und': 1, 'hose': 1, 'tisch': 1, 'wiese': 1, 'glas': 1, 'baum': 1 }; const QUESTION_ONE_QUESTIONS = ['teller', 'hund', 'lampe', 'brief', 'apfel', 'hose', 'tisch', 'wiese', 'glas', 'baum']; const QUESTION_TWO_ANSWERS = {}; var QUESTION_TWO_QUESTIONS; const QUESTION_THREE_QUESTIONS_PT1 = ['7, 2', '4, 7, 9', '5, 4, 9, 6', '2, 7, 5, 3, 6', '8, 1, 3, 5, 4, 2']; const QUESTION_THREE_QUESTIONS_PT2 = ['8, 6', '3, 1, 5', '1, 9, 7, 4', '1, 3, 5, 4, 8', '4, 1, 2, 7, 9, 5']; const QUESTION_THREE_ANSWERS_PT1 = ['27', '974', '6945', '63572', '245318']; const QUESTION_THREE_ANSWERS_PT2 = ['68', '513', '4791', '84531', '597214']; LoadQuestionTwo(); function LoadQuestionTwo () { var xmlhttp; if (window.XMLHttpRequest) { // code for IE7+, Firefox, Chrome, Opera, Safari xmlhttp = new XMLHttpRequest(); } else { // code for IE6, IE5 xmlhttp = new ActiveXObject('Microsoft.XMLHTTP'); } xmlhttp.onreadystatechange = function () { if (xmlhttp.readyState === 4 && xmlhttp.status === 200) { var text = xmlhttp.responseText.toLowerCase(); // Now convert it into array using regex QUESTION_TWO_QUESTIONS = text.split('\r\n'); for (let word of QUESTION_TWO_QUESTIONS) { QUESTION_TWO_ANSWERS[word] = 1; } } }; xmlhttp.open('GET', 'lebensmittel.txt', true); xmlhttp.send(); } // #endregion // #region points const questionPoints = { 1: 0, 2: 0, 3: 0, 4: 0, 5: 0 }; // #endregion // tokenization const separators = [' ', '\\\+', '-', '\\\(', '\\\)', '\\*', '/', ':', '\\\?']; // Timers var timerId; // #region html elements var serverPara = document.querySelector('.server'); var diagnosticPara = document.querySelector('.output'); var testBtn = document.querySelector('button'); var testBtn2 = document.getElementById('speechBtn'); var infoPara = document.getElementById('info'); var questionNumDisplay = document.querySelector('.quest'); // #endregion // websocket to communicate with the server var ws = new WebSocket('wss://' + window.location.host + window.location.pathname + 'ws'); // #region speech recognition initialization var recognition = new SpeechRecognition(); recognition.lang = 'de-DE'; // recognition.interimResults = false; recognition.maxAlternatives = 1; recognition.continuous = true; var answerQuery = ''; var proceedWithRecording = false; // #endregion // #region speech synthesis initialization var speechsynth = new SpeechSynthesisUtterance(); var listSpeechsynth = new SpeechSynthesisUtterance(); var voices; // #endregion // #region speech events window.speechSynthesis.onvoiceschanged = function () { voices = window.speechSynthesis.getVoices(); voices.forEach(element => { if (element.name === 'Google Deutsch') { speechsynth.voice = element; listSpeechsynth.voice = element; } }); speechsynth.rate = 0.88; listSpeechsynth.rate = 0.7; }; speechsynth.onend = function (event) { if (proceedWithRecording) { recognition.start(); console.log('reocgnition started. Question: ' + question); } proceedWithRecording = false; diagnosticPara.textContent = ''; console.log('global speech end'); }; // #endregion // #region websocket events ws.onopen = function () { serverPara.style.background = 'green'; serverPara.innerHTML = 'Server online'; }; ws.onmessage = function (payload) { var dialogflowResult = JSON.parse(payload.data); checkIntent(dialogflowResult); }; // #endregion // INTENT HANDLING function checkIntent (result) { switch (result.intent.displayName) { case QUIT_INTENT: state = 'quit'; stopRecognitionFallback(); speak('Beende die Durchführung.'); break; case WELCOME_INTENT: state = 'detect'; proceedWithRecording = true; speak(result.fulfillmentText); break; case WELCOME_FOLLOWUP_YES: startQuestion(1); break; case WELCOME_FOLLOWUP_NO: speak('Alles klar. Danke fürs Benutzen.'); break; /// /// /// currently disabled feature // case MORE_TIME: // state = 'detect'; // proceedWithRecording = true; // speak('Brauchen Sie noch etwas Zeit?'); // break; // case MORE_TIME_YES: // rePrompt = true; // state = 'answer'; // proceedWithRecording = true; // speak('Alles klar'); // break; // case MORE_TIME_NO: // state = 'answer'; // speak('Verstanden'); // recognition.stop(); // ws.send(answerQuery); // break; // case CHOOSE_QUESTION: // question = result.parameters.fields.num.numberValue; // state = 'answer'; // handleQuestion(); // break; /// /// /// case HELP_INTENT: stopRecognitionFallback(); recognition.stop(); startQuestion(question); break; case FALLBACK_INTENT: // fallback what should happen if no intent is detected // this is handled in the processSpeech() function break; default: break; } } // #region question handling function startQuestion (number) { question = number; state = 'answer'; answerQuery = ''; questionNumDisplay.textContent = 'Question: ' + question; handleQuestion(); } function handleQuestion () { switch (question) { case 1: readQuestionOne(QUESTION_ONE); break; case 2: readQuestionTwo(); break; case 3: readQuestionThree(); break; case 4: readQuestionFour(); break; case 5: proceedWithRecording = true; speak(QUESTION_FIVE); break; } } function readQuestionOne (questionText) { speak(questionText); for (let i = 0; i < QUESTION_ONE_QUESTIONS.length; i++) { let utterance = new SpeechSynthesisUtterance(); utterance.voice = voices[2]; utterance.rate = 0.75; utterance.text = QUESTION_ONE_QUESTIONS[i]; window.speechSynthesis.speak(utterance); if (i === 9) { utterance.onend = function (event) { recognition.start(); console.log('reocgnition started. Question: ' + question); }; } } } function readQuestionTwo () { let utterance = new SpeechSynthesisUtterance(); utterance.voice = voices[2]; utterance.text = QUESTION_TWO; utterance.rate = 0.8; window.speechSynthesis.speak(utterance); utterance.onend = function (event) { window.setTimeout( function () { recognition.stop(); window.setTimeout( function () { handleAnswer(answerQuery); answerQuery = ''; }, 3000); }, 6000); recognition.start(); console.log('reocgnition started. Question: ' + question); }; } function readQuestionThree () { speak('Dankeschön, weiter geht es mit der nächsten Frage.'); let utterance = new SpeechSynthesisUtterance(); utterance.voice = voices[2]; utterance.text = QUESTION_THREE; utterance.rate = 0.88; window.speechSynthesis.speak(utterance); utterance.onend = function (event) { proceedWithRecording = true; speak(QUESTION_THREE_QUESTIONS_PT1[questionThreeCount]); }; } function readQuestionFour () { speak(QUESTION_FOUR); for (let i = 0; i < QUESTION_FOUR_PT2.length; i++) { let utterance = new SpeechSynthesisUtterance(); utterance.voice = voices[2]; utterance.rate = 0.75; utterance.text = QUESTION_FOUR_PT2[i]; window.speechSynthesis.speak(utterance); } speak('Sie haben hierfür 1 Minute Zeit.'); } function handleAnswer (query) { switch (question) { case 1: handleAnswerToFirstQuestion(query); break; case 2: handleAnswerToSecondQuestion(query); break; case 3: handleAnswerToThirdQuestion(query); break; case 4: // not possible with the current state of web speech api // handleAnswerToFourthQuestion(query); break; case 5: handleAnswerToFifthQuestion(query); break; } } function handleAnswerToFirstQuestion (answer) { deconstructAndCalculateAnswer(answer); if (partTwo) { partTwo = false; speak('Vielen Dank, nun geht es weiter mit der nächsten Frage'); startQuestion(2); } else { // currently disabled feature // rePrompt = false; speak(QUESTION_ONE_PT2); readQuestionOne(QUESTION_ONE_PT2); partTwo = true; } } function handleAnswerToSecondQuestion (answer) { deconstructAndCalculateAnswer(answer); startQuestion(3); } function handleAnswerToThirdQuestion (query) { speechsynth.rate = 0.88; query = query.replace(' ', ''); let answerArray; let questionArray; if (!partTwo) { answerArray = QUESTION_THREE_ANSWERS_PT1; } else { answerArray = QUESTION_THREE_ANSWERS_PT2; } if (query === answerArray[questionThreeCount]) { strike = 0; partTwo = false; questionThreeCount++; questionPoints[question] = questionThreeCount + 1; questionArray = QUESTION_THREE_QUESTIONS_PT1; } else { strike++; partTwo = true; questionArray = QUESTION_THREE_QUESTIONS_PT2; } if (strike === 2 || questionThreeCount === 5) { speechsynth.rate = 0.88; console.log('question 3 points: ' + questionPoints[question]); speak('weiter geht es mit der Nächsten Frage'); // Question 4 is not possible to take, because of technical limitation // startQuestion(4); startQuestion(5); return; } proceedWithRecording = true; speak(questionArray[questionThreeCount]); console.log('count: ' + questionThreeCount + ', strike: ' + strike + ', points: ' + questionPoints[question]); } // not possible with the current state of web speech api // function handleAnswerToFourthQuestion (answer) { // // In order to evaluate the answer, the recognizer has to distinguish between the number 77 spoken "seventy seven" and 7-7 spoken "seven seven", which it doesnt // startQuestion(5); // } function handleAnswerToFifthQuestion (answer) { deconstructAndCalculateAnswer(answer); speak(FINISH); } // #endregion // NOT SUPPORTED // function recognizeSpeech () { // var arr; // switch (question) { // case 1: // arr = QUESTION_ONE_QUESTIONS; // break; // case 2: // arr = QUESTION_TWO_QUESTIONS; // break; // case 3: // arr = [1, 2, 3, 4, 5, 6, 7, 8, 9]; // break; // case 4: // arr = ['eins', 'zwei', 'drei', 'vier', 'fünf', 'sechs', 'sieben', 'acht', 'neun']; // break; // case 5: // break; // } // var grammar = '#JSGF V1.0; grammar arr; public = ' + arr.join(' | ') + ' ;'; // var speechRecognitionList = new SpeechGrammarList(); // speechRecognitionList.addFromString(grammar, 1); // recognition.grammars = speechRecognitionList; // recognition.start(); // } // #region speech recognition event recognition.onresult = function (event) { var last = event.results.length - 1; var speechResult = event.results[last][0].transcript.toLowerCase(); diagnosticPara.textContent += speechResult + ' '; // console.log('Confidence: ' + event.results[0][0].confidence) console.log('process: ' + speechResult); processSpeech(speechResult); // testBtn.disabled = false // testBtn.textContent = 'record...' }; recognition.onspeechend = function () { // recognition.stop(); // testBtn.disabled = false; // testBtn.textContent = 'Start new test'; }; recognition.onerror = function (event) { testBtn.disabled = false; testBtn.textContent = 'Start new test'; diagnosticPara.textContent = 'Error occurred in recognition: ' + event.error; }; recognition.onaudiostart = function (event) { // Fired when the user agent has started to capture audio. }; recognition.onaudioend = function (event) { }; recognition.onend = function (event) { // Fired when the speech recognition service has disconnected. }; recognition.onnomatch = function (event) { // Fired when the speech recognition service returns a final result with no significant recognition. This may involve some degree of recognition, which doesn't meet or exceed the confidence threshold. // console.log('SpeechRecognition.onnomatch') }; recognition.onsoundstart = function (event) { // Fired when any sound — recognisable speech or not — has been detected. }; recognition.onsoundend = function (event) { // Fired when any sound — recognisable speech or not — has stopped being detected. }; recognition.onspeechstart = function (event) { // Fired when sound that is recognised by the speech recognition service as speech has been detected. }; recognition.onstart = function (event) { // Fired when the speech recognition service has begun listening to incoming audio with intent to recognize grammars associated with the current SpeechRecognition. }; // } // #endregion // #region global functions function processSpeech (speechResult) { console.log('To dialogflow: ' + speechResult); ws.send(speechResult); let timeOut; switch (question) { case 1: timeOut = 6500; break; case 2: answerQuery += speechResult; return; case 3: if (speechResult.includes('uhr')) { speechResult = speechResult.replace('uhr', ''); } timeOut = 6500; break; case 4: break; case 5: timeOut = 6500; break; } if (state === 'answer') { stopRecognitionFallback(); answerQuery += speechResult; timerId = window.setTimeout( function () { // currently disabled // if (!rePrompt) { // ws.send('ich brauche noch etwas Zeit') // } else { console.log('recording end. Evaluate: ' + answerQuery); handleAnswer(answerQuery); diagnosticPara.textContent = ''; // } recognition.stop(); console.log('timer fallback'); }, timeOut); } else { console.log('recording end.'); recognition.stop(); } } function startDemenzScreening () { // ws.send('starte demenz test'); startQuestion(4); testBtn.disabled = true; testBtn.textContent = 'Test in progress'; infoPara.textContent = 'wait...'; diagnosticPara.textContent = 'detecting...'; } function testSpeechOut () { question = 4; } function speak (sentence) { speechsynth.text = sentence; window.speechSynthesis.speak(speechsynth); } function stopRecognitionFallback () { if (timerId != undefined) { clearTimeout(timerId); } } function deconstructAndCalculateAnswer (answer) { let questionAnswers; switch (question) { case 1: questionAnswers = QUESTION_ONE_ANSWERS; break; case 2: questionAnswers = QUESTION_TWO_ANSWERS; break; case 5: questionAnswers = QUESTION_ONE_ANSWERS; break; } var tokens = answer.split(new RegExp(separators.join('|'), 'g')); questionPoints[question] += calculatePoints(tokens, questionAnswers); } function calculatePoints (tokens, d) { let points = 0; let dict = {}; Object.assign(dict, d); for (let word of tokens) { if (dict[word] !== undefined) { points += dict[word]; delete dict[word]; } } return points; } // #endregion testBtn.addEventListener('click', startDemenzScreening); testBtn2.addEventListener('click', testSpeechOut);