no message
This commit is contained in:
Niko
@@ -0,0 +1,32 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
|
||||
<meta name="viewport" content="width=device-width">
|
||||
|
||||
<title>Wort matcher</title>
|
||||
|
||||
<link rel="stylesheet" href="style.css">
|
||||
<!--[if lt IE 9]>
|
||||
<script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
|
||||
<![endif]-->
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<h1>Speech Recognition</h1>
|
||||
<p id=info>Press the button to activate microphone.</p>
|
||||
<p id=query hidden>query</p>
|
||||
|
||||
<button>Start new test</button>
|
||||
<button id="speechBtn">Speech</button>
|
||||
<button onclick="getElementById('query').innerHTML = 'changed'">Change query</button>
|
||||
|
||||
<div>
|
||||
<p class="server">Server offline</p>
|
||||
<p class="output" style="font-style: italic;">... erkannte worte ...</p>
|
||||
</div>
|
||||
|
||||
<script src="ws-client.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,54 @@
|
||||
body, html {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
html {
|
||||
height: 100%;
|
||||
background-color: teal;
|
||||
}
|
||||
|
||||
body {
|
||||
height: inherit;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
h1, p {
|
||||
font-family: sans-serif;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
div p {
|
||||
padding: 20px;
|
||||
background-color: rgba(0,0,0,0.2);
|
||||
}
|
||||
|
||||
div {
|
||||
overflow: auto;
|
||||
position: absolute;
|
||||
bottom: 0;
|
||||
right: 0;
|
||||
left: 0;
|
||||
}
|
||||
|
||||
button {
|
||||
margin: 0 auto;
|
||||
display: block;
|
||||
font-size: 1.1rem;
|
||||
width: 170px;
|
||||
line-height: 2;
|
||||
margin-top: 30px;
|
||||
}
|
||||
|
||||
@media all and (max-height: 410px) {
|
||||
div {
|
||||
position: static;
|
||||
}
|
||||
}
|
||||
|
||||
.phrase {
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.output {
|
||||
font-style: italic;
|
||||
}
|
||||
@@ -0,0 +1,509 @@
|
||||
// web speech recognition api
|
||||
var SpeechRecognition = SpeechRecognition || webkitSpeechRecognition;
|
||||
var SpeechGrammarList = SpeechGrammarList || webkitSpeechGrammarList;
|
||||
var SpeechRecognitionEvent = SpeechRecognitionEvent || webkitSpeechRecognitionEvent;
|
||||
|
||||
// state management
|
||||
var state = '';
|
||||
var question = 0;
|
||||
var rePrompt = false;
|
||||
var partTwo = false;
|
||||
var count = 0;
|
||||
var strike = 0;
|
||||
|
||||
// Questions
|
||||
const QUESTION_ONE = 'Ich werde Ihnen jetzt langsam eine Liste mit Worten vorlesen. Danach wiederholen Sie bitte möglichst viele dieser Worte. Auf die Reihenfolge kommt es nicht an.';
|
||||
const QUESTION_ONE_PT2 = 'Vielen Dank. Nun nenne ich Ihnen die gleichen 10 Worte ein zweites mal. Auch danach sollen Sie wieder möglichst viele Worte wiederholen';
|
||||
const QUESTION_TWO = 'Nennen Sie mir bitte so viel Dinge wie möglich, die man im Supermarkt kaufen kann. Sie haben dafür eine Minute Zeit.';
|
||||
const QUESTION_THREE = 'Ich werde Ihnen jetzt eine Zahlenreihe nennen, die Sie mir dann bitte in umgekehrter Reihenfolge wiederholen sollen. Wenn ich beispielsweise, vier - fünf sage, dann sagen Sie bitte, fünf - vier.';
|
||||
|
||||
// intents
|
||||
const WELCOME_INTENT = 'Default Welcome Intent';
|
||||
const WELCOME_FOLLOWUP_YES = 'Default Welcome Intent - yes';
|
||||
const WELCOME_FOLLOWUP_NO = 'Default Welcome Intent - no';
|
||||
const MORE_TIME = 'Add Time Intent';
|
||||
const MORE_TIME_YES = 'Add Time Intent - yes';
|
||||
const MORE_TIME_NO = 'Add Time Intent - no';
|
||||
const QUIT_INTENT = 'Quit Intent';
|
||||
const FALLBACK_INTENT = 'Default Fallback Intent';
|
||||
const HELP_INTENT = 'Help Intent';
|
||||
const CHOOSE_QUESTION = 'Frage_Starten';
|
||||
const NEXT_QUESTION = 'Nächste Frage';
|
||||
|
||||
// question one results
|
||||
const QUESTION_ONE_ANSWERS = ['teller', 'hund', 'lampe', 'brief', 'apfel', 'apfelwiese', 'apfelwiese', 'apfelbaum', 'apfelbaum', 'und', 'hose', 'tisch', 'wiese', 'glas', 'baum'];
|
||||
const QUESTION_ONE_QUESTIONS = ['teller', 'hund', 'lampe', 'brief', 'apfel', 'hose', 'tisch', 'wiese', 'glas', 'baum'];
|
||||
const QUESTION_TWO_ANSWERS = [''];
|
||||
const QUESTION_TWO_QUESTIONS = [''];
|
||||
const QUESTION_THREE_QUESTIONS_PT1 = ['7, 2', '4, 7, 9', '5, 4, 9, 6', '2, 7, 5, 3, 6', '8, 1, 3, 5, 4, 2'];
|
||||
const QUESTION_THREE_QUESTIONS_PT2 = ['8, 6', '3, 1, 5', '1, 9, 7, 4', '1, 3, 5, 4, 8', '4, 1, 2, 7, 9, 5'];
|
||||
const QUESTION_THREE_ANSWERS_PT1 = ['27', '974', '6945', '63572', '245318'];
|
||||
const QUESTION_THREE_ANSWERS_PT2 = ['68', '513', '4791', '84531', '597214'];
|
||||
|
||||
// points
|
||||
var questionOnePoints = 0;
|
||||
var questionTwoPoints = 0;
|
||||
var questionThreePoints = 0;
|
||||
var questionFourPoints = 0;
|
||||
var questionFivePoints = 0;
|
||||
var questionSixPoints = 0;
|
||||
|
||||
// tokenization
|
||||
const separators = [' ', '\\\+', '-', '\\\(', '\\\)', '\\*', '/', ':', '\\\?'];
|
||||
|
||||
// Timers
|
||||
var timerId;
|
||||
|
||||
// html elements
|
||||
var serverPara = document.querySelector('.server');
|
||||
var diagnosticPara = document.querySelector('.output');
|
||||
var testBtn = document.querySelector('button');
|
||||
var testBtn2 = document.getElementById('speechBtn');
|
||||
var infoPara = document.getElementById('info');
|
||||
var userPrompt = document.getElementById('query');
|
||||
|
||||
console.log(window.location.host + window.location.pathname);
|
||||
|
||||
// websocket to communicate with the server
|
||||
var ws = new WebSocket('ws://localhost:8000/ws');
|
||||
|
||||
// speech recognition
|
||||
var recognition = new SpeechRecognition();
|
||||
recognition.lang = 'de-DE';
|
||||
// recognition.interimResults = false;
|
||||
recognition.maxAlternatives = 1;
|
||||
recognition.continuous = true;
|
||||
var answerQuery = '';
|
||||
var skipRecording = false;
|
||||
|
||||
// speech synthesis
|
||||
var speechsynth = new SpeechSynthesisUtterance();
|
||||
var listSpeechsynth = new SpeechSynthesisUtterance();
|
||||
var voices;
|
||||
|
||||
window.speechSynthesis.onvoiceschanged = function () {
|
||||
voices = window.speechSynthesis.getVoices();
|
||||
voices.forEach(element => {
|
||||
if (element.name === 'Google Deutsch') {
|
||||
speechsynth.voice = element;
|
||||
listSpeechsynth.voice = element;
|
||||
}
|
||||
});
|
||||
|
||||
listSpeechsynth.rate = 0.7;
|
||||
};
|
||||
|
||||
function startDemenzScreening () {
|
||||
console.log('button clicked');
|
||||
ws.send('starte demenz test');
|
||||
//
|
||||
// state = 'answer'
|
||||
// recognizeSpeech()
|
||||
|
||||
//
|
||||
testBtn.disabled = true;
|
||||
testBtn.textContent = 'Test in progress';
|
||||
infoPara.textContent = 'wait...';
|
||||
}
|
||||
|
||||
function speak (sentence) {
|
||||
speechsynth.text = sentence;
|
||||
window.speechSynthesis.speak(speechsynth);
|
||||
}
|
||||
|
||||
function testSpeechOut () {
|
||||
console.log('click');
|
||||
// skipRecording = true
|
||||
// let utterance = new SpeechSynthesisUtterance()
|
||||
// utterance.voice = voices[2]
|
||||
// // utterance.rate = 0.75
|
||||
// utterance.text = QUESTION_THREE
|
||||
// window.speechSynthesis.speak(utterance)
|
||||
//
|
||||
// question = 3;
|
||||
// startQuestion(question);
|
||||
// state = 'answer';
|
||||
speechsynth.text = 'test 123';
|
||||
speechsynth.volume = 1;
|
||||
speechsynth.rate = 1;
|
||||
console.log(speechsynth);
|
||||
window.speechSynthesis.speak(speechsynth);
|
||||
console.log(window.speechSynthesis);
|
||||
// speak('Niko ist der größte hurn eu west');
|
||||
}
|
||||
|
||||
// websocket events
|
||||
ws.onopen = function () {
|
||||
serverPara.style.background = 'green';
|
||||
serverPara.innerHTML = 'Server online';
|
||||
};
|
||||
|
||||
ws.onmessage = function (payload) {
|
||||
var dialogflowResult = JSON.parse(payload.data);
|
||||
checkIntent(dialogflowResult);
|
||||
document.querySelector('h1').innerHTML = dialogflowResult.intent.displayName;
|
||||
};
|
||||
|
||||
// INTENT HANDLING
|
||||
function checkIntent (result) {
|
||||
switch (result.intent.displayName) {
|
||||
case QUIT_INTENT:
|
||||
state = 'quit';
|
||||
skipRecording = true;
|
||||
speak('Okay, Danke fürs Benutzen.');
|
||||
break;
|
||||
case WELCOME_INTENT:
|
||||
state = 'detect';
|
||||
// speak(result.fulfillmentText)
|
||||
speak('go?');
|
||||
break;
|
||||
case WELCOME_FOLLOWUP_YES:
|
||||
startQuestion(1);
|
||||
break;
|
||||
case WELCOME_FOLLOWUP_NO:
|
||||
skipRecording = true;
|
||||
speak('Okay, Danke fürs Benutzen.');
|
||||
break;
|
||||
case MORE_TIME:
|
||||
state = 'detect';
|
||||
speak('Brauchen Sie noch etwas Zeit?');
|
||||
break;
|
||||
case MORE_TIME_YES:
|
||||
rePrompt = true;
|
||||
state = 'answer';
|
||||
speak('Alles klar');
|
||||
break;
|
||||
case MORE_TIME_NO:
|
||||
skipRecording = true;
|
||||
state = 'answer';
|
||||
speak('Verstanden');
|
||||
recognition.stop();
|
||||
ws.send(answerQuery);
|
||||
break;
|
||||
case CHOOSE_QUESTION:
|
||||
question = result.parameters.fields.num.numberValue;
|
||||
state = 'answer';
|
||||
handleQuestion();
|
||||
break;
|
||||
case FALLBACK_INTENT:
|
||||
// if (state === 'answer') {
|
||||
// handleAnswer(result.queryText)
|
||||
// }
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
function startQuestion (number) {
|
||||
question = number;
|
||||
state = 'answer';
|
||||
handleQuestion();
|
||||
}
|
||||
|
||||
// QUESTION HANDLING
|
||||
function handleQuestion () {
|
||||
switch (question) {
|
||||
case 1:
|
||||
readQuestionOne(QUESTION_ONE);
|
||||
break;
|
||||
case 2:
|
||||
readQuestionTwo(QUESTION_TWO);
|
||||
break;
|
||||
case 3:
|
||||
// skipRecording = true;
|
||||
// speak(QUESTION_THREE);
|
||||
speak(QUESTION_THREE_QUESTIONS_PT1[0]);
|
||||
break;
|
||||
case 4:
|
||||
break;
|
||||
case 5:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
function handleAnswer (query) {
|
||||
switch (question) {
|
||||
case 1:
|
||||
handleAnswerToFirstQuestion(query);
|
||||
break;
|
||||
case 2:
|
||||
handleAnswerToSecondQuestion(query);
|
||||
break;
|
||||
case 3:
|
||||
handleAnswerToThirdQuestion(query);
|
||||
break;
|
||||
case 4:
|
||||
break;
|
||||
case 5:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
function handleAnswerToFirstQuestion (answer) {
|
||||
var tokens = answer.split(new RegExp(separators.join('|'), 'g'));
|
||||
questionOnePoints = calculatePoints(tokens, QUESTION_ONE_ANSWERS);
|
||||
if (partTwo) {
|
||||
partTwo = false;
|
||||
startQuestion(2);
|
||||
// state = 'detect'
|
||||
} else {
|
||||
rePrompt = false;
|
||||
speak(QUESTION_ONE_PT2);
|
||||
partTwo = true;
|
||||
}
|
||||
}
|
||||
|
||||
function handleAnswerToSecondQuestion (answer) {
|
||||
var tokens = answer.split(new RegExp(separators.join('|'), 'g'));
|
||||
questionTwoPoints = calculatePoints(tokens, QUESTION_TWO_ANSWERS);
|
||||
skipRecording = true;
|
||||
speak('Sie haben ' + questionOnePoints + 'Punkte');
|
||||
startQuestion(3);
|
||||
// state = 'detect'
|
||||
}
|
||||
|
||||
function handleAnswerToThirdQuestion (query) {
|
||||
speechsynth.rate = 0.87;
|
||||
query = query.replace(' ', '');
|
||||
|
||||
let answerArray;
|
||||
let questionArray;
|
||||
if (!partTwo) {
|
||||
answerArray = QUESTION_THREE_ANSWERS_PT1;
|
||||
} else {
|
||||
answerArray = QUESTION_THREE_ANSWERS_PT2;
|
||||
}
|
||||
if (query === answerArray[count]) {
|
||||
strike = 0;
|
||||
partTwo = false;
|
||||
count++;
|
||||
questionThreePoints = count + 1;
|
||||
questionArray = QUESTION_THREE_QUESTIONS_PT1;
|
||||
} else {
|
||||
strike++;
|
||||
partTwo = true;
|
||||
questionArray = QUESTION_THREE_QUESTIONS_PT2;
|
||||
}
|
||||
|
||||
if (strike === 2 || count === 5) {
|
||||
speechsynth.rate = 1;
|
||||
skipRecording = true;
|
||||
speak('weiter geht es mit der Nächsten Frage');
|
||||
startQuestion(4);
|
||||
return;
|
||||
}
|
||||
|
||||
speak(questionArray[count]);
|
||||
|
||||
console.log('count: ' + count + ', strike: ' + strike + ', points: ' + questionThreePoints);
|
||||
}
|
||||
|
||||
// Question specific functions
|
||||
function readQuestionOne (text) {
|
||||
skipRecording = true;
|
||||
speak(text);
|
||||
for (let i = 0; i < QUESTION_ONE_QUESTIONS.length; i++) {
|
||||
let utterance = new SpeechSynthesisUtterance();
|
||||
utterance.voice = voices[2];
|
||||
utterance.rate = 0.75;
|
||||
utterance.text = QUESTION_ONE_QUESTIONS[i];
|
||||
window.speechSynthesis.speak(utterance);
|
||||
if (i === 9) {
|
||||
utterance.onend = function (event) {
|
||||
console.log('end of aufzählung' + i);
|
||||
recognizeSpeech();
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function readQuestionTwo (text) {
|
||||
let utterance = new SpeechSynthesisUtterance();
|
||||
utterance.voice = voices[2];
|
||||
utterance.text = text;
|
||||
window.speechSynthesis.speak(utterance);
|
||||
console.log('q 2 started');
|
||||
utterance.onend = function (event) {
|
||||
window.setTimeout(
|
||||
function () {
|
||||
recognition.stop();
|
||||
handleAnswer(answerQuery);
|
||||
}, 60000);
|
||||
console.log('q 2 recognition started');
|
||||
recognizeSpeech();
|
||||
};
|
||||
}
|
||||
|
||||
function calculatePoints (tokens, array) {
|
||||
let points = 0;
|
||||
for (let i of array) {
|
||||
for (let j of tokens) {
|
||||
if (i === j) {
|
||||
points++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return points;
|
||||
}
|
||||
|
||||
speechsynth.onend = function (event) {
|
||||
switch (question) {
|
||||
case 1:
|
||||
break;
|
||||
case 2:
|
||||
break;
|
||||
case 3:
|
||||
break;
|
||||
case 4:
|
||||
break;
|
||||
case 5:
|
||||
break;
|
||||
}
|
||||
if (!skipRecording) {
|
||||
recognizeSpeech();
|
||||
}
|
||||
skipRecording = false;
|
||||
console.log('global speech end');
|
||||
};
|
||||
|
||||
function recognizeSpeech () {
|
||||
if (state === 'answer') {
|
||||
var arr;
|
||||
switch (question) {
|
||||
case 1:
|
||||
arr = QUESTION_ONE_QUESTIONS;
|
||||
break;
|
||||
case 2:
|
||||
return;
|
||||
case 3:
|
||||
arr = [1, 2, 3, 4, 5, 6, 7, 8, 9];
|
||||
break;
|
||||
case 4:
|
||||
break;
|
||||
case 5:
|
||||
break;
|
||||
}
|
||||
|
||||
var grammar = '#JSGF V1.0; grammar colors; public <color> = ' + arr.join(' | ') + ' ;';
|
||||
var speechRecognitionList = new SpeechGrammarList();
|
||||
speechRecognitionList.addFromString(grammar, 1);
|
||||
recognition.grammars = speechRecognitionList;
|
||||
}
|
||||
recognition.start();
|
||||
console.log('reocgnition started');
|
||||
|
||||
recognition.onresult = function (event) {
|
||||
var last = event.results.length - 1;
|
||||
var speechResult = event.results[last][0].transcript.toLowerCase();
|
||||
|
||||
diagnosticPara.textContent += speechResult + ' ';
|
||||
|
||||
// console.log('Confidence: ' + event.results[0][0].confidence)
|
||||
|
||||
processSpeech(speechResult);
|
||||
// testBtn.disabled = false
|
||||
// testBtn.textContent = 'record...'
|
||||
};
|
||||
|
||||
function processSpeech (speechResult) {
|
||||
console.log('To dialogflow: ' + speechResult);
|
||||
ws.send(speechResult);
|
||||
|
||||
let timeOut;
|
||||
switch (question) {
|
||||
case 1:
|
||||
timeOut = 6500;
|
||||
break;
|
||||
case 2:
|
||||
answerQuery += speechResult;
|
||||
return;
|
||||
case 3:
|
||||
if (speechResult.includes('uhr')) {
|
||||
speechResult = speechResult.replace('uhr', '');
|
||||
}
|
||||
timeOut = 6500;
|
||||
break;
|
||||
case 4:
|
||||
break;
|
||||
case 5:
|
||||
break;
|
||||
}
|
||||
|
||||
if (state === 'answer') {
|
||||
if (timerId != undefined) {
|
||||
clearTimeout(timerId);
|
||||
}
|
||||
answerQuery += speechResult;
|
||||
timerId = window.setTimeout(
|
||||
function () {
|
||||
// if (!rePrompt) {
|
||||
// ws.send('ich brauche noch etwas Zeit')
|
||||
// } else {
|
||||
console.log('recording end. Evaluate: ' + answerQuery);
|
||||
handleAnswer(answerQuery);
|
||||
answerQuery = '';
|
||||
diagnosticPara.textContent = '';
|
||||
// }
|
||||
recognition.stop();
|
||||
console.log('timer fallback');
|
||||
}, timeOut);
|
||||
} else {
|
||||
console.log('recording end.');
|
||||
recognition.stop();
|
||||
}
|
||||
}
|
||||
|
||||
recognition.onspeechend = function () {
|
||||
// recognition.stop();
|
||||
// testBtn.disabled = false;
|
||||
// testBtn.textContent = 'Start new test';
|
||||
};
|
||||
|
||||
recognition.onerror = function (event) {
|
||||
testBtn.disabled = false;
|
||||
testBtn.textContent = 'Start new test';
|
||||
diagnosticPara.textContent = 'Error occurred in recognition: ' + event.error;
|
||||
};
|
||||
|
||||
recognition.onaudiostart = function (event) {
|
||||
// Fired when the user agent has started to capture audio.
|
||||
|
||||
};
|
||||
|
||||
recognition.onaudioend = function (event) {
|
||||
|
||||
};
|
||||
|
||||
recognition.onend = function (event) {
|
||||
// Fired when the speech recognition service has disconnected.
|
||||
|
||||
};
|
||||
|
||||
recognition.onnomatch = function (event) {
|
||||
// Fired when the speech recognition service returns a final result with no significant recognition. This may involve some degree of recognition, which doesn't meet or exceed the confidence threshold.
|
||||
// console.log('SpeechRecognition.onnomatch')
|
||||
};
|
||||
|
||||
recognition.onsoundstart = function (event) {
|
||||
// Fired when any sound — recognisable speech or not — has been detected.
|
||||
|
||||
};
|
||||
|
||||
recognition.onsoundend = function (event) {
|
||||
// Fired when any sound — recognisable speech or not — has stopped being detected.
|
||||
|
||||
};
|
||||
|
||||
recognition.onspeechstart = function (event) {
|
||||
// Fired when sound that is recognised by the speech recognition service as speech has been detected.
|
||||
|
||||
};
|
||||
recognition.onstart = function (event) {
|
||||
// Fired when the speech recognition service has begun listening to incoming audio with intent to recognize grammars associated with the current SpeechRecognition.
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
testBtn.addEventListener('click', startDemenzScreening);
|
||||
testBtn2.addEventListener('click', testSpeechOut);
|
||||
Reference in New Issue
Block a user