// #region web speech recognition api var SpeechRecognition = SpeechRecognition || webkitSpeechRecognition; var SpeechGrammarList = SpeechGrammarList || webkitSpeechGrammarList; var SpeechRecognitionEvent = SpeechRecognitionEvent || webkitSpeechRecognitionEvent; // #endregion // #region state management var state = ''; var question = 0; var rePrompt = false; var partTwo = false; var questionThreeCount = 0; var strike = 0; // #endregion // #region questions const QUESTION_ONE = 'Ich werde Ihnen jetzt langsam eine Liste mit Worten vorlesen. Danach wiederholen Sie bitte möglichst viele dieser Worte. Auf die Reihenfolge kommt es nicht an.'; const QUESTION_ONE_PT2 = 'Vielen Dank. Nun nenne ich Ihnen die gleichen 10 Worte ein zweites mal. Auch danach sollen Sie wieder möglichst viele Worte wiederholen'; const QUESTION_TWO = 'Nennen Sie mir bitte so viel Dinge wie möglich, die man im Supermarkt kaufen kann. Sie haben dafür eine Minute Zeit. Und Los'; const QUESTION_THREE = 'Ich werde Ihnen jetzt eine Zahlenreihe nennen, die Sie mir dann bitte in umgekehrter Reihenfolge wiederholen sollen. Wenn ich beispielsweise, vier - fünf sage, dann sagen Sie bitte, fünf - vier.'; // #endregion // #region intents const WELCOME_INTENT = 'Default Welcome Intent'; const WELCOME_FOLLOWUP_YES = 'Default Welcome Intent - yes'; const WELCOME_FOLLOWUP_NO = 'Default Welcome Intent - no'; const MORE_TIME = 'Add Time Intent'; const MORE_TIME_YES = 'Add Time Intent - yes'; const MORE_TIME_NO = 'Add Time Intent - no'; const QUIT_INTENT = 'Quit Intent'; const FALLBACK_INTENT = 'Default Fallback Intent'; const HELP_INTENT = 'Help Intent'; const CHOOSE_QUESTION = 'Frage_Starten'; const NEXT_QUESTION = 'Nächste Frage'; // #endregion // #region questions and expected results const QUESTION_ONE_ANSWERS = { 'teller': 1, 'hund': 1, 'lampe': 1, 'brief': 1, 'apfel': 1, 'apfelwiese': 2, 'apfelbaum': 2, 'und': 1, 'hose': 1, 'tisch': 1, 'wiese': 1, 'glas': 1, 'baum': 1 }; const QUESTION_ONE_QUESTIONS = ['teller', 'hund', 'lampe', 'brief', 'apfel', 'hose', 'tisch', 'wiese', 'glas', 'baum']; const QUESTION_TWO_ANSWERS = {}; var QUESTION_TWO_QUESTIONS; const QUESTION_THREE_QUESTIONS_PT1 = ['7, 2', '4, 7, 9', '5, 4, 9, 6', '2, 7, 5, 3, 6', '8, 1, 3, 5, 4, 2']; const QUESTION_THREE_QUESTIONS_PT2 = ['8, 6', '3, 1, 5', '1, 9, 7, 4', '1, 3, 5, 4, 8', '4, 1, 2, 7, 9, 5']; const QUESTION_THREE_ANSWERS_PT1 = ['27', '974', '6945', '63572', '245318']; const QUESTION_THREE_ANSWERS_PT2 = ['68', '513', '4791', '84531', '597214']; LoadQuestionTwo(); function LoadQuestionTwo () { var xmlhttp; if (window.XMLHttpRequest) { // code for IE7+, Firefox, Chrome, Opera, Safari xmlhttp = new XMLHttpRequest(); } else { // code for IE6, IE5 xmlhttp = new ActiveXObject('Microsoft.XMLHTTP'); } xmlhttp.onreadystatechange = function () { if (xmlhttp.readyState === 4 && xmlhttp.status === 200) { var text = xmlhttp.responseText.toLowerCase(); // Now convert it into array using regex QUESTION_TWO_QUESTIONS = text.split('\r\n'); for (let word of QUESTION_TWO_QUESTIONS) { QUESTION_TWO_ANSWERS[word] = 1; } } }; xmlhttp.open('GET', 'lebensmittel.txt', true); xmlhttp.send(); } // #endregion // #region points const questionPoints = { 1: 0, 2: 0, 3: 0, 4: 0, 5: 0 }; // #endregion // tokenization const separators = [' ', '\\\+', '-', '\\\(', '\\\)', '\\*', '/', ':', '\\\?']; // Timers var timerId; // #region html elements var serverPara = document.querySelector('.server'); var diagnosticPara = document.querySelector('.output'); var testBtn = document.querySelector('button'); var testBtn2 = document.getElementById('speechBtn'); var infoPara = document.getElementById('info'); var userPrompt = document.getElementById('query'); // #endregion // websocket to communicate with the server var ws = new WebSocket('ws://' + window.location.host + window.location.pathname + 'ws'); // #region speech recognition initialization var recognition = new SpeechRecognition(); recognition.lang = 'de-DE'; // recognition.interimResults = false; recognition.maxAlternatives = 1; recognition.continuous = true; var answerQuery = ''; var skipRecording = false; // #endregion // #region speech synthesis initialization var speechsynth = new SpeechSynthesisUtterance(); var listSpeechsynth = new SpeechSynthesisUtterance(); var voices; // #endregion // #region speech events window.speechSynthesis.onvoiceschanged = function () { voices = window.speechSynthesis.getVoices(); voices.forEach(element => { if (element.name === 'Google Deutsch') { speechsynth.voice = element; listSpeechsynth.voice = element; } }); listSpeechsynth.rate = 0.7; }; speechsynth.onend = function (event) { switch (question) { case 1: break; case 2: break; case 3: break; case 4: break; case 5: break; } if (!skipRecording) { recognition.start(); console.log('reocgnition started. Question: ' + question); } skipRecording = false; diagnosticPara.textContent = ''; console.log('global speech end'); }; // #endregion // #region websocket events ws.onopen = function () { serverPara.style.background = 'green'; serverPara.innerHTML = 'Server online'; }; ws.onmessage = function (payload) { var dialogflowResult = JSON.parse(payload.data); checkIntent(dialogflowResult); document.querySelector('h1').innerHTML = dialogflowResult.intent.displayName; }; // #endregion // INTENT HANDLING function checkIntent (result) { switch (result.intent.displayName) { case QUIT_INTENT: state = 'quit'; if (timerId !== undefined) { clearTimeout(timerId); } skipRecording = true; speak('Beende die Durchführung.'); break; case WELCOME_INTENT: state = 'detect'; speak(result.fulfillmentText); break; case WELCOME_FOLLOWUP_YES: startQuestion(1); break; case WELCOME_FOLLOWUP_NO: skipRecording = true; speak('Okay, Danke fürs Benutzen.'); break; case MORE_TIME: state = 'detect'; speak('Brauchen Sie noch etwas Zeit?'); break; case MORE_TIME_YES: rePrompt = true; state = 'answer'; speak('Alles klar'); break; case MORE_TIME_NO: skipRecording = true; state = 'answer'; speak('Verstanden'); recognition.stop(); ws.send(answerQuery); break; case CHOOSE_QUESTION: question = result.parameters.fields.num.numberValue; state = 'answer'; handleQuestion(); break; case FALLBACK_INTENT: // if (state === 'answer') { // handleAnswer(result.queryText) // } break; default: break; } } // #region question handling function startQuestion (number) { question = number; state = 'answer'; handleQuestion(); } function handleQuestion () { switch (question) { case 1: skipRecording = true; speak(QUESTION_ONE); readQuestionOne(); break; case 2: readQuestionTwo(); break; case 3: readQuestionThree(); break; case 4: break; case 5: break; } } function readQuestionOne () { for (let i = 0; i < QUESTION_ONE_QUESTIONS.length; i++) { let utterance = new SpeechSynthesisUtterance(); utterance.voice = voices[2]; utterance.rate = 0.75; utterance.text = QUESTION_ONE_QUESTIONS[i]; window.speechSynthesis.speak(utterance); if (i === 9) { utterance.onend = function (event) { recognition.start(); console.log('reocgnition started. Question: ' + question); }; } } } function readQuestionTwo () { let utterance = new SpeechSynthesisUtterance(); utterance.voice = voices[2]; utterance.text = QUESTION_TWO; window.speechSynthesis.speak(utterance); utterance.onend = function (event) { window.setTimeout( function () { recognition.stop(); console.log('recognition stopped'); handleAnswer(answerQuery); }, 6000); recognition.start(); console.log('reocgnition started. Question: ' + question); }; } function readQuestionThree () { skipRecording = true; speak('Dankeschön, weiter geht es mit der nächsten Frage.'); let utterance = new SpeechSynthesisUtterance(); utterance.voice = voices[2]; utterance.text = QUESTION_THREE; window.speechSynthesis.speak(utterance); utterance.onend = function (event) { speak(QUESTION_THREE_QUESTIONS_PT1[questionThreeCount]); }; utterance.onerror = function (event) { console.log('An error has occurred with the speech synthesis: ' + event.error); }; } function handleAnswer (query) { switch (question) { case 1: handleAnswerToFirstQuestion(query); break; case 2: handleAnswerToSecondQuestion(query); break; case 3: handleAnswerToThirdQuestion(query); break; case 4: break; case 5: break; } } function handleAnswerToFirstQuestion (answer) { var tokens = answer.split(new RegExp(separators.join('|'), 'g')); questionPoints[question] += calculatePoints(tokens, QUESTION_ONE_ANSWERS); if (partTwo) { partTwo = false; console.log('question 1 points: ' + questionPoints[question]); skipRecording = true; speak('Vielen Dank, nun geht es weiter mit der nächsten Frage'); startQuestion(2); // state = 'detect' } else { rePrompt = false; skipRecording = true; speak(QUESTION_ONE_PT2); readQuestionOne(QUESTION_ONE); partTwo = true; } } function handleAnswerToSecondQuestion (answer) { var tokens = answer.split(new RegExp(separators.join('|'), 'g')); questionPoints[question] = calculatePoints(tokens, QUESTION_TWO_ANSWERS); console.log('question 2 points: ' + questionPoints[question]); startQuestion(3); // state = 'detect' } function handleAnswerToThirdQuestion (query) { speechsynth.rate = 0.87; query = query.replace(' ', ''); let answerArray; let questionArray; if (!partTwo) { answerArray = QUESTION_THREE_ANSWERS_PT1; } else { answerArray = QUESTION_THREE_ANSWERS_PT2; } if (query === answerArray[questionThreeCount]) { strike = 0; partTwo = false; questionThreeCount++; questionPoints[question] = questionThreeCount + 1; questionArray = QUESTION_THREE_QUESTIONS_PT1; } else { strike++; partTwo = true; questionArray = QUESTION_THREE_QUESTIONS_PT2; } if (strike === 2 || questionThreeCount === 5) { speechsynth.rate = 1; console.log('question 3 points: ' + questionPoints[question]); skipRecording = true; speak('weiter geht es mit der Nächsten Frage'); startQuestion(4); return; } speak(questionArray[questionThreeCount]); console.log('count: ' + questionThreeCount + ', strike: ' + strike + ', points: ' + questionPoints[question]); } // #endregion // #region global functions function startDemenzScreening () { ws.send('starte demenz test'); // startQuestion(2); testBtn.disabled = true; testBtn.textContent = 'Test in progress'; infoPara.textContent = 'wait...'; diagnosticPara.textContent = 'detecting...'; } function speak (sentence) { speechsynth.text = sentence; window.speechSynthesis.speak(speechsynth); } function testSpeechOut () { answerQuery = 'apfel wiese tisch apfel lampe pferd'; question = 1; for (let i = 0; i < 2; i++) { var tokens = answerQuery.split(new RegExp(separators.join('|'), 'g')); questionPoints[question] += calculatePoints(tokens, QUESTION_ONE_ANSWERS); } console.log(questionPoints[question]); // speechsynth.text = 'test 123'; // speechsynth.volume = 1; // speechsynth.rate = 1; // console.log(speechsynth); // window.speechSynthesis.speak(speechsynth); // console.log(window.speechSynthesis); } // function recognizeSpeech () { // if (state === 'answer') { // var arr; // switch (question) { // case 1: // arr = QUESTION_ONE_QUESTIONS; // break; // case 2: // // arr = QUESTION_TWO_QUESTIONS; // break; // case 3: // arr = [1, 2, 3, 4, 5, 6, 7, 8, 9]; // break; // case 4: // break; // case 5: // break; // } // // var grammar = '#JSGF V1.0; grammar colors; public = ' + arr.join(' | ') + ' ;'; // // var speechRecognitionList = new SpeechGrammarList(); // // speechRecognitionList.addFromString(grammar, 1); // // recognition.grammars = speechRecognitionList; // } recognition.onresult = function (event) { var last = event.results.length - 1; var speechResult = event.results[last][0].transcript.toLowerCase(); diagnosticPara.textContent += speechResult + ' '; // console.log('Confidence: ' + event.results[0][0].confidence) console.log('process: ' + speechResult); processSpeech(speechResult); // testBtn.disabled = false // testBtn.textContent = 'record...' }; function processSpeech (speechResult) { console.log('To dialogflow: ' + speechResult); ws.send(speechResult); let timeOut; switch (question) { case 1: timeOut = 6500; break; case 2: answerQuery += speechResult; return; case 3: if (speechResult.includes('uhr')) { speechResult = speechResult.replace('uhr', ''); } timeOut = 6500; break; case 4: break; case 5: timeOut = 6500; break; } if (state === 'answer') { if (timerId != undefined) { clearTimeout(timerId); } answerQuery += speechResult; timerId = window.setTimeout( function () { // if (!rePrompt) { // ws.send('ich brauche noch etwas Zeit') // } else { console.log('recording end. Evaluate: ' + answerQuery); handleAnswer(answerQuery); answerQuery = ''; diagnosticPara.textContent = ''; // } recognition.stop(); console.log('timer fallback'); }, timeOut); } else { console.log('recording end.'); recognition.stop(); } } // #region speech recognition event recognition.onspeechend = function () { // recognition.stop(); // testBtn.disabled = false; // testBtn.textContent = 'Start new test'; }; recognition.onerror = function (event) { testBtn.disabled = false; testBtn.textContent = 'Start new test'; diagnosticPara.textContent = 'Error occurred in recognition: ' + event.error; }; recognition.onaudiostart = function (event) { // Fired when the user agent has started to capture audio. }; recognition.onaudioend = function (event) { }; recognition.onend = function (event) { // Fired when the speech recognition service has disconnected. }; recognition.onnomatch = function (event) { // Fired when the speech recognition service returns a final result with no significant recognition. This may involve some degree of recognition, which doesn't meet or exceed the confidence threshold. // console.log('SpeechRecognition.onnomatch') }; recognition.onsoundstart = function (event) { // Fired when any sound — recognisable speech or not — has been detected. }; recognition.onsoundend = function (event) { // Fired when any sound — recognisable speech or not — has stopped being detected. }; recognition.onspeechstart = function (event) { // Fired when sound that is recognised by the speech recognition service as speech has been detected. }; recognition.onstart = function (event) { // Fired when the speech recognition service has begun listening to incoming audio with intent to recognize grammars associated with the current SpeechRecognition. }; // } // #endregion function calculatePoints (tokens, d) { let points = 0; let dict = {}; Object.assign(dict, d); for (let word of tokens) { if (dict[word] !== undefined) { points += dict[word]; delete dict[word]; } } return points; } // #endregion testBtn.addEventListener('click', startDemenzScreening); testBtn2.addEventListener('click', testSpeechOut);