// web speech recognition api var SpeechRecognition = SpeechRecognition || webkitSpeechRecognition; var SpeechGrammarList = SpeechGrammarList || webkitSpeechGrammarList; var SpeechRecognitionEvent = SpeechRecognitionEvent || webkitSpeechRecognitionEvent; // state management var state = ''; var question = 0; var rePrompt = false; var partTwo = false; var count = 0; var strike = 0; // Questions const QUESTION_ONE = 'Ich werde Ihnen jetzt langsam eine Liste mit Worten vorlesen. Danach wiederholen Sie bitte möglichst viele dieser Worte. Auf die Reihenfolge kommt es nicht an.'; const QUESTION_ONE_PT2 = 'Vielen Dank. Nun nenne ich Ihnen die gleichen 10 Worte ein zweites mal. Auch danach sollen Sie wieder möglichst viele Worte wiederholen'; const QUESTION_TWO = 'Nennen Sie mir bitte so viel Dinge wie möglich, die man im Supermarkt kaufen kann. Sie haben dafür eine Minute Zeit.'; const QUESTION_THREE = 'Ich werde Ihnen jetzt eine Zahlenreihe nennen, die Sie mir dann bitte in umgekehrter Reihenfolge wiederholen sollen. Wenn ich beispielsweise, vier - fünf sage, dann sagen Sie bitte, fünf - vier.'; // intents const WELCOME_INTENT = 'Default Welcome Intent'; const WELCOME_FOLLOWUP_YES = 'Default Welcome Intent - yes'; const WELCOME_FOLLOWUP_NO = 'Default Welcome Intent - no'; const MORE_TIME = 'Add Time Intent'; const MORE_TIME_YES = 'Add Time Intent - yes'; const MORE_TIME_NO = 'Add Time Intent - no'; const QUIT_INTENT = 'Quit Intent'; const FALLBACK_INTENT = 'Default Fallback Intent'; const HELP_INTENT = 'Help Intent'; const CHOOSE_QUESTION = 'Frage_Starten'; const NEXT_QUESTION = 'Nächste Frage'; // question one results const QUESTION_ONE_ANSWERS = ['teller', 'hund', 'lampe', 'brief', 'apfel', 'apfelwiese', 'apfelwiese', 'apfelbaum', 'apfelbaum', 'und', 'hose', 'tisch', 'wiese', 'glas', 'baum']; const QUESTION_ONE_QUESTIONS = ['teller', 'hund', 'lampe', 'brief', 'apfel', 'hose', 'tisch', 'wiese', 'glas', 'baum']; const QUESTION_TWO_ANSWERS = ['']; const QUESTION_TWO_QUESTIONS = ['']; const QUESTION_THREE_QUESTIONS_PT1 = ['7, 2', '4, 7, 9', '5, 4, 9, 6', '2, 7, 5, 3, 6', '8, 1, 3, 5, 4, 2']; const QUESTION_THREE_QUESTIONS_PT2 = ['8, 6', '3, 1, 5', '1, 9, 7, 4', '1, 3, 5, 4, 8', '4, 1, 2, 7, 9, 5']; const QUESTION_THREE_ANSWERS_PT1 = ['27', '974', '6945', '63572', '245318']; const QUESTION_THREE_ANSWERS_PT2 = ['68', '513', '4791', '84531', '597214']; // points var questionOnePoints = 0; var questionTwoPoints = 0; var questionThreePoints = 0; var questionFourPoints = 0; var questionFivePoints = 0; var questionSixPoints = 0; // tokenization const separators = [' ', '\\\+', '-', '\\\(', '\\\)', '\\*', '/', ':', '\\\?']; // Timers var timerId; // html elements var serverPara = document.querySelector('.server'); var diagnosticPara = document.querySelector('.output'); var testBtn = document.querySelector('button'); var testBtn2 = document.getElementById('speechBtn'); var infoPara = document.getElementById('info'); var userPrompt = document.getElementById('query'); console.log(window.location.host + window.location.pathname); // websocket to communicate with the server var ws = new WebSocket('ws://localhost:8000/ws'); // speech recognition var recognition = new SpeechRecognition(); recognition.lang = 'de-DE'; // recognition.interimResults = false; recognition.maxAlternatives = 1; recognition.continuous = true; var answerQuery = ''; var skipRecording = false; // speech synthesis var speechsynth = new SpeechSynthesisUtterance(); var listSpeechsynth = new SpeechSynthesisUtterance(); var voices; window.speechSynthesis.onvoiceschanged = function () { voices = window.speechSynthesis.getVoices(); voices.forEach(element => { if (element.name === 'Google Deutsch') { speechsynth.voice = element; listSpeechsynth.voice = element; } }); listSpeechsynth.rate = 0.7; }; function startDemenzScreening () { console.log('button clicked'); ws.send('starte demenz test'); // // state = 'answer' // recognizeSpeech() // testBtn.disabled = true; testBtn.textContent = 'Test in progress'; infoPara.textContent = 'wait...'; } function speak (sentence) { speechsynth.text = sentence; window.speechSynthesis.speak(speechsynth); } function testSpeechOut () { console.log('click'); // skipRecording = true // let utterance = new SpeechSynthesisUtterance() // utterance.voice = voices[2] // // utterance.rate = 0.75 // utterance.text = QUESTION_THREE // window.speechSynthesis.speak(utterance) // // question = 3; // startQuestion(question); // state = 'answer'; speechsynth.text = 'test 123'; speechsynth.volume = 1; speechsynth.rate = 1; console.log(speechsynth); window.speechSynthesis.speak(speechsynth); console.log(window.speechSynthesis); // speak('Niko ist der größte hurn eu west'); } // websocket events ws.onopen = function () { serverPara.style.background = 'green'; serverPara.innerHTML = 'Server online'; }; ws.onmessage = function (payload) { var dialogflowResult = JSON.parse(payload.data); checkIntent(dialogflowResult); document.querySelector('h1').innerHTML = dialogflowResult.intent.displayName; }; // INTENT HANDLING function checkIntent (result) { switch (result.intent.displayName) { case QUIT_INTENT: state = 'quit'; skipRecording = true; speak('Okay, Danke fürs Benutzen.'); break; case WELCOME_INTENT: state = 'detect'; // speak(result.fulfillmentText) speak('go?'); break; case WELCOME_FOLLOWUP_YES: startQuestion(1); break; case WELCOME_FOLLOWUP_NO: skipRecording = true; speak('Okay, Danke fürs Benutzen.'); break; case MORE_TIME: state = 'detect'; speak('Brauchen Sie noch etwas Zeit?'); break; case MORE_TIME_YES: rePrompt = true; state = 'answer'; speak('Alles klar'); break; case MORE_TIME_NO: skipRecording = true; state = 'answer'; speak('Verstanden'); recognition.stop(); ws.send(answerQuery); break; case CHOOSE_QUESTION: question = result.parameters.fields.num.numberValue; state = 'answer'; handleQuestion(); break; case FALLBACK_INTENT: // if (state === 'answer') { // handleAnswer(result.queryText) // } break; default: break; } } function startQuestion (number) { question = number; state = 'answer'; handleQuestion(); } // QUESTION HANDLING function handleQuestion () { switch (question) { case 1: readQuestionOne(QUESTION_ONE); break; case 2: readQuestionTwo(QUESTION_TWO); break; case 3: // skipRecording = true; // speak(QUESTION_THREE); speak(QUESTION_THREE_QUESTIONS_PT1[0]); break; case 4: break; case 5: break; } } function handleAnswer (query) { switch (question) { case 1: handleAnswerToFirstQuestion(query); break; case 2: handleAnswerToSecondQuestion(query); break; case 3: handleAnswerToThirdQuestion(query); break; case 4: break; case 5: break; } } function handleAnswerToFirstQuestion (answer) { var tokens = answer.split(new RegExp(separators.join('|'), 'g')); questionOnePoints = calculatePoints(tokens, QUESTION_ONE_ANSWERS); if (partTwo) { partTwo = false; startQuestion(2); // state = 'detect' } else { rePrompt = false; speak(QUESTION_ONE_PT2); partTwo = true; } } function handleAnswerToSecondQuestion (answer) { var tokens = answer.split(new RegExp(separators.join('|'), 'g')); questionTwoPoints = calculatePoints(tokens, QUESTION_TWO_ANSWERS); skipRecording = true; speak('Sie haben ' + questionOnePoints + 'Punkte'); startQuestion(3); // state = 'detect' } function handleAnswerToThirdQuestion (query) { speechsynth.rate = 0.87; query = query.replace(' ', ''); let answerArray; let questionArray; if (!partTwo) { answerArray = QUESTION_THREE_ANSWERS_PT1; } else { answerArray = QUESTION_THREE_ANSWERS_PT2; } if (query === answerArray[count]) { strike = 0; partTwo = false; count++; questionThreePoints = count + 1; questionArray = QUESTION_THREE_QUESTIONS_PT1; } else { strike++; partTwo = true; questionArray = QUESTION_THREE_QUESTIONS_PT2; } if (strike === 2 || count === 5) { speechsynth.rate = 1; skipRecording = true; speak('weiter geht es mit der Nächsten Frage'); startQuestion(4); return; } speak(questionArray[count]); console.log('count: ' + count + ', strike: ' + strike + ', points: ' + questionThreePoints); } // Question specific functions function readQuestionOne (text) { skipRecording = true; speak(text); for (let i = 0; i < QUESTION_ONE_QUESTIONS.length; i++) { let utterance = new SpeechSynthesisUtterance(); utterance.voice = voices[2]; utterance.rate = 0.75; utterance.text = QUESTION_ONE_QUESTIONS[i]; window.speechSynthesis.speak(utterance); if (i === 9) { utterance.onend = function (event) { console.log('end of aufzählung' + i); recognizeSpeech(); }; } } } function readQuestionTwo (text) { let utterance = new SpeechSynthesisUtterance(); utterance.voice = voices[2]; utterance.text = text; window.speechSynthesis.speak(utterance); console.log('q 2 started'); utterance.onend = function (event) { window.setTimeout( function () { recognition.stop(); handleAnswer(answerQuery); }, 60000); console.log('q 2 recognition started'); recognizeSpeech(); }; } function calculatePoints (tokens, array) { let points = 0; for (let i of array) { for (let j of tokens) { if (i === j) { points++; } } } return points; } speechsynth.onend = function (event) { switch (question) { case 1: break; case 2: break; case 3: break; case 4: break; case 5: break; } if (!skipRecording) { recognizeSpeech(); } skipRecording = false; console.log('global speech end'); }; function recognizeSpeech () { if (state === 'answer') { var arr; switch (question) { case 1: arr = QUESTION_ONE_QUESTIONS; break; case 2: return; case 3: arr = [1, 2, 3, 4, 5, 6, 7, 8, 9]; break; case 4: break; case 5: break; } var grammar = '#JSGF V1.0; grammar colors; public = ' + arr.join(' | ') + ' ;'; var speechRecognitionList = new SpeechGrammarList(); speechRecognitionList.addFromString(grammar, 1); recognition.grammars = speechRecognitionList; } recognition.start(); console.log('reocgnition started'); recognition.onresult = function (event) { var last = event.results.length - 1; var speechResult = event.results[last][0].transcript.toLowerCase(); diagnosticPara.textContent += speechResult + ' '; // console.log('Confidence: ' + event.results[0][0].confidence) processSpeech(speechResult); // testBtn.disabled = false // testBtn.textContent = 'record...' }; function processSpeech (speechResult) { console.log('To dialogflow: ' + speechResult); ws.send(speechResult); let timeOut; switch (question) { case 1: timeOut = 6500; break; case 2: answerQuery += speechResult; return; case 3: if (speechResult.includes('uhr')) { speechResult = speechResult.replace('uhr', ''); } timeOut = 6500; break; case 4: break; case 5: break; } if (state === 'answer') { if (timerId != undefined) { clearTimeout(timerId); } answerQuery += speechResult; timerId = window.setTimeout( function () { // if (!rePrompt) { // ws.send('ich brauche noch etwas Zeit') // } else { console.log('recording end. Evaluate: ' + answerQuery); handleAnswer(answerQuery); answerQuery = ''; diagnosticPara.textContent = ''; // } recognition.stop(); console.log('timer fallback'); }, timeOut); } else { console.log('recording end.'); recognition.stop(); } } recognition.onspeechend = function () { // recognition.stop(); // testBtn.disabled = false; // testBtn.textContent = 'Start new test'; }; recognition.onerror = function (event) { testBtn.disabled = false; testBtn.textContent = 'Start new test'; diagnosticPara.textContent = 'Error occurred in recognition: ' + event.error; }; recognition.onaudiostart = function (event) { // Fired when the user agent has started to capture audio. }; recognition.onaudioend = function (event) { }; recognition.onend = function (event) { // Fired when the speech recognition service has disconnected. }; recognition.onnomatch = function (event) { // Fired when the speech recognition service returns a final result with no significant recognition. This may involve some degree of recognition, which doesn't meet or exceed the confidence threshold. // console.log('SpeechRecognition.onnomatch') }; recognition.onsoundstart = function (event) { // Fired when any sound — recognisable speech or not — has been detected. }; recognition.onsoundend = function (event) { // Fired when any sound — recognisable speech or not — has stopped being detected. }; recognition.onspeechstart = function (event) { // Fired when sound that is recognised by the speech recognition service as speech has been detected. }; recognition.onstart = function (event) { // Fired when the speech recognition service has begun listening to incoming audio with intent to recognize grammars associated with the current SpeechRecognition. }; } testBtn.addEventListener('click', startDemenzScreening); testBtn2.addEventListener('click', testSpeechOut);