You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

ws-client.js 15KB

5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
5 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544
  1. // #region web speech recognition api
  2. var SpeechRecognition = SpeechRecognition || webkitSpeechRecognition;
  3. var SpeechGrammarList = SpeechGrammarList || webkitSpeechGrammarList;
  4. var SpeechRecognitionEvent = SpeechRecognitionEvent || webkitSpeechRecognitionEvent;
  5. // #endregion
  6. // #region state management
  7. var state = '';
  8. var question = 0;
  9. var rePrompt = false;
  10. var partTwo = false;
  11. var questionThreeCount = 0;
  12. var strike = 0;
  13. // #endregion
  14. // #region questions
  15. const QUESTION_ONE = 'Ich werde Ihnen jetzt langsam eine Liste mit Worten vorlesen. Danach wiederholen Sie bitte möglichst viele dieser Worte. Auf die Reihenfolge kommt es nicht an.';
  16. const QUESTION_ONE_PT2 = 'Vielen Dank. Nun nenne ich Ihnen die gleichen 10 Worte ein zweites mal. Auch danach sollen Sie wieder möglichst viele Worte wiederholen';
  17. const QUESTION_TWO = 'Nennen Sie mir bitte so viel Dinge wie möglich, die man im Supermarkt kaufen kann. Sie haben dafür eine Minute Zeit. Und Los';
  18. const QUESTION_THREE = 'Ich werde Ihnen jetzt eine Zahlenreihe nennen, die Sie mir dann bitte in umgekehrter Reihenfolge wiederholen sollen. Wenn ich beispielsweise, vier - fünf sage, dann sagen Sie bitte, fünf - vier.';
  19. // #endregion
  20. // #region intents
  21. const WELCOME_INTENT = 'Default Welcome Intent';
  22. const WELCOME_FOLLOWUP_YES = 'Default Welcome Intent - yes';
  23. const WELCOME_FOLLOWUP_NO = 'Default Welcome Intent - no';
  24. const MORE_TIME = 'Add Time Intent';
  25. const MORE_TIME_YES = 'Add Time Intent - yes';
  26. const MORE_TIME_NO = 'Add Time Intent - no';
  27. const QUIT_INTENT = 'Quit Intent';
  28. const FALLBACK_INTENT = 'Default Fallback Intent';
  29. const HELP_INTENT = 'Help Intent';
  30. const CHOOSE_QUESTION = 'Frage_Starten';
  31. const NEXT_QUESTION = 'Nächste Frage';
  32. // #endregion
  33. // #region questions and expected results
  34. const QUESTION_ONE_ANSWERS = { 'teller': 1, 'hund': 1, 'lampe': 1, 'brief': 1, 'apfel': 1, 'apfelwiese': 2, 'apfelbaum': 2, 'und': 1, 'hose': 1, 'tisch': 1, 'wiese': 1, 'glas': 1, 'baum': 1 };
  35. const QUESTION_ONE_QUESTIONS = ['teller', 'hund', 'lampe', 'brief', 'apfel', 'hose', 'tisch', 'wiese', 'glas', 'baum'];
  36. const QUESTION_TWO_ANSWERS = {};
  37. var QUESTION_TWO_QUESTIONS = [''];
  38. const QUESTION_THREE_QUESTIONS_PT1 = ['7, 2', '4, 7, 9', '5, 4, 9, 6', '2, 7, 5, 3, 6', '8, 1, 3, 5, 4, 2'];
  39. const QUESTION_THREE_QUESTIONS_PT2 = ['8, 6', '3, 1, 5', '1, 9, 7, 4', '1, 3, 5, 4, 8', '4, 1, 2, 7, 9, 5'];
  40. const QUESTION_THREE_ANSWERS_PT1 = ['27', '974', '6945', '63572', '245318'];
  41. const QUESTION_THREE_ANSWERS_PT2 = ['68', '513', '4791', '84531', '597214'];
  42. LoadQuestionTwo();
  43. function LoadQuestionTwo () {
  44. var xmlhttp;
  45. if (window.XMLHttpRequest) { // code for IE7+, Firefox, Chrome, Opera, Safari
  46. xmlhttp = new XMLHttpRequest();
  47. } else { // code for IE6, IE5
  48. xmlhttp = new ActiveXObject('Microsoft.XMLHTTP');
  49. }
  50. xmlhttp.onreadystatechange = function () {
  51. if (xmlhttp.readyState === 4 && xmlhttp.status === 200) {
  52. var text = xmlhttp.responseText.toLowerCase();
  53. // Now convert it into array using regex
  54. QUESTION_TWO_QUESTIONS = text.split('\r\n');
  55. for (let word of QUESTION_TWO_QUESTIONS) {
  56. QUESTION_TWO_ANSWERS[word] = 1;
  57. }
  58. }
  59. };
  60. xmlhttp.open('GET', 'lebensmittel.txt', true);
  61. xmlhttp.send();
  62. }
  63. // #endregion
  64. // #region points
  65. var questionOnePoints = 0;
  66. var questionTwoPoints = 0;
  67. var questionThreePoints = 0;
  68. var questionFourPoints = 0;
  69. var questionFivePoints = 0;
  70. var questionSixPoints = 0;
  71. // #endregion
  72. // tokenization
  73. const separators = [' ', '\\\+', '-', '\\\(', '\\\)', '\\*', '/', ':', '\\\?'];
  74. // Timers
  75. var timerId;
  76. // #region html elements
  77. var serverPara = document.querySelector('.server');
  78. var diagnosticPara = document.querySelector('.output');
  79. var testBtn = document.querySelector('button');
  80. var testBtn2 = document.getElementById('speechBtn');
  81. var infoPara = document.getElementById('info');
  82. var userPrompt = document.getElementById('query');
  83. // #endregion
  84. // websocket to communicate with the server
  85. var ws = new WebSocket('ws://' + window.location.host + window.location.pathname + 'ws');
  86. // #region speech recognition initialization
  87. var recognition = new SpeechRecognition();
  88. recognition.lang = 'de-DE';
  89. // recognition.interimResults = false;
  90. recognition.maxAlternatives = 1;
  91. recognition.continuous = true;
  92. var answerQuery = '';
  93. var skipRecording = false;
  94. // #endregion
  95. // #region speech synthesis initialization
  96. var speechsynth = new SpeechSynthesisUtterance();
  97. var listSpeechsynth = new SpeechSynthesisUtterance();
  98. var voices;
  99. // #endregion
  100. // #region speech events
  101. window.speechSynthesis.onvoiceschanged = function () {
  102. voices = window.speechSynthesis.getVoices();
  103. voices.forEach(element => {
  104. if (element.name === 'Google Deutsch') {
  105. speechsynth.voice = element;
  106. listSpeechsynth.voice = element;
  107. }
  108. });
  109. listSpeechsynth.rate = 0.7;
  110. };
  111. speechsynth.onend = function (event) {
  112. switch (question) {
  113. case 1:
  114. break;
  115. case 2:
  116. break;
  117. case 3:
  118. break;
  119. case 4:
  120. break;
  121. case 5:
  122. break;
  123. }
  124. if (!skipRecording) {
  125. recognizeSpeech();
  126. }
  127. skipRecording = false;
  128. diagnosticPara = '';
  129. console.log('global speech end');
  130. };
  131. // #endregion
  132. // #region websocket events
  133. ws.onopen = function () {
  134. serverPara.style.background = 'green';
  135. serverPara.innerHTML = 'Server online';
  136. };
  137. ws.onmessage = function (payload) {
  138. var dialogflowResult = JSON.parse(payload.data);
  139. checkIntent(dialogflowResult);
  140. document.querySelector('h1').innerHTML = dialogflowResult.intent.displayName;
  141. };
  142. // #endregion
  143. // INTENT HANDLING
  144. function checkIntent (result) {
  145. switch (result.intent.displayName) {
  146. case QUIT_INTENT:
  147. state = 'quit';
  148. if (timerId !== undefined) {
  149. clearTimeout(timerId);
  150. }
  151. skipRecording = true;
  152. speak('Beende die Durchführung.');
  153. break;
  154. case WELCOME_INTENT:
  155. state = 'detect';
  156. // speak(result.fulfillmentText)
  157. speak('go');
  158. break;
  159. case WELCOME_FOLLOWUP_YES:
  160. startQuestion(2);
  161. break;
  162. case WELCOME_FOLLOWUP_NO:
  163. skipRecording = true;
  164. speak('Okay, Danke fürs Benutzen.');
  165. break;
  166. case MORE_TIME:
  167. state = 'detect';
  168. speak('Brauchen Sie noch etwas Zeit?');
  169. break;
  170. case MORE_TIME_YES:
  171. rePrompt = true;
  172. state = 'answer';
  173. speak('Alles klar');
  174. break;
  175. case MORE_TIME_NO:
  176. skipRecording = true;
  177. state = 'answer';
  178. speak('Verstanden');
  179. recognition.stop();
  180. ws.send(answerQuery);
  181. break;
  182. case CHOOSE_QUESTION:
  183. question = result.parameters.fields.num.numberValue;
  184. state = 'answer';
  185. handleQuestion();
  186. break;
  187. case FALLBACK_INTENT:
  188. // if (state === 'answer') {
  189. // handleAnswer(result.queryText)
  190. // }
  191. break;
  192. default:
  193. break;
  194. }
  195. }
  196. // #region question handling
  197. function startQuestion (number) {
  198. question = number;
  199. state = 'answer';
  200. handleQuestion();
  201. }
  202. function handleQuestion () {
  203. switch (question) {
  204. case 1:
  205. skipRecording = true;
  206. speak(QUESTION_ONE);
  207. readQuestionOne();
  208. break;
  209. case 2:
  210. readQuestionTwo();
  211. break;
  212. case 3:
  213. readQuestionThree();
  214. break;
  215. case 4:
  216. break;
  217. case 5:
  218. break;
  219. }
  220. }
  221. function readQuestionOne () {
  222. for (let i = 0; i < QUESTION_ONE_QUESTIONS.length; i++) {
  223. let utterance = new SpeechSynthesisUtterance();
  224. utterance.voice = voices[2];
  225. utterance.rate = 0.75;
  226. utterance.text = QUESTION_ONE_QUESTIONS[i];
  227. window.speechSynthesis.speak(utterance);
  228. if (i === 9) {
  229. utterance.onend = function (event) {
  230. recognizeSpeech();
  231. };
  232. }
  233. }
  234. }
  235. function readQuestionTwo () {
  236. let utterance = new SpeechSynthesisUtterance();
  237. utterance.voice = voices[2];
  238. utterance.text = QUESTION_TWO;
  239. window.speechSynthesis.speak(utterance);
  240. utterance.onend = function (event) {
  241. window.setTimeout(
  242. function () {
  243. recognition.stop();
  244. handleAnswer(answerQuery);
  245. }, 6000);
  246. recognizeSpeech();
  247. };
  248. }
  249. function readQuestionThree () {
  250. recognition = false;
  251. speak('Dankeschön. Weiter geht es mit der nächsten Frage. ');
  252. let utterance = new SpeechSynthesisUtterance();
  253. utterance.voice = voices[2];
  254. utterance.text = QUESTION_THREE;
  255. window.speechSynthesis.speak(utterance);
  256. utterance.onend = function (event) {
  257. console.log('speach end');
  258. speak(QUESTION_THREE_QUESTIONS_PT1[questionThreeCount]);
  259. };
  260. utterance.onerror = function (event) {
  261. console.log('An error has occurred with the speech synthesis: ' + event.error);
  262. };
  263. }
  264. function handleAnswer (query) {
  265. switch (question) {
  266. case 1:
  267. handleAnswerToFirstQuestion(query);
  268. break;
  269. case 2:
  270. handleAnswerToSecondQuestion(query);
  271. break;
  272. case 3:
  273. handleAnswerToThirdQuestion(query);
  274. break;
  275. case 4:
  276. break;
  277. case 5:
  278. break;
  279. }
  280. }
  281. function handleAnswerToFirstQuestion (answer) {
  282. var tokens = answer.split(new RegExp(separators.join('|'), 'g'));
  283. questionOnePoints += calculatePoints(tokens, QUESTION_ONE_ANSWERS);
  284. if (partTwo) {
  285. partTwo = false;
  286. skipRecording = true;
  287. speak('Vielen Dank, nun geht es weiter mit der nächsten Frage');
  288. startQuestion(2);
  289. // state = 'detect'
  290. } else {
  291. rePrompt = false;
  292. skipRecording = true;
  293. speak(QUESTION_ONE_PT2);
  294. readQuestionOne(QUESTION_ONE);
  295. partTwo = true;
  296. }
  297. }
  298. function handleAnswerToSecondQuestion (answer) {
  299. var tokens = answer.split(new RegExp(separators.join('|'), 'g'));
  300. questionTwoPoints = calculatePoints(tokens, QUESTION_TWO_ANSWERS);
  301. startQuestion(3);
  302. // state = 'detect'
  303. }
  304. function handleAnswerToThirdQuestion (query) {
  305. speechsynth.rate = 0.87;
  306. query = query.replace(' ', '');
  307. let answerArray;
  308. let questionArray;
  309. if (!partTwo) {
  310. answerArray = QUESTION_THREE_ANSWERS_PT1;
  311. } else {
  312. answerArray = QUESTION_THREE_ANSWERS_PT2;
  313. }
  314. if (query === answerArray[questionThreeCount]) {
  315. strike = 0;
  316. partTwo = false;
  317. questionThreeCount++;
  318. questionThreePoints = questionThreeCount + 1;
  319. questionArray = QUESTION_THREE_QUESTIONS_PT1;
  320. } else {
  321. strike++;
  322. partTwo = true;
  323. questionArray = QUESTION_THREE_QUESTIONS_PT2;
  324. }
  325. if (strike === 2 || questionThreeCount === 5) {
  326. speechsynth.rate = 1;
  327. skipRecording = true;
  328. speak('weiter geht es mit der Nächsten Frage');
  329. startQuestion(4);
  330. return;
  331. }
  332. speak(questionArray[questionThreeCount]);
  333. console.log('count: ' + questionThreeCount + ', strike: ' + strike + ', points: ' + questionThreePoints);
  334. }
  335. // #endregion
  336. // #region global functions
  337. function startDemenzScreening () {
  338. // ws.send('starte demenz test');
  339. startQuestion(3);
  340. testBtn.disabled = true;
  341. testBtn.textContent = 'Test in progress';
  342. infoPara.textContent = 'wait...';
  343. diagnosticPara.textContent = 'detecting...';
  344. }
  345. function speak (sentence) {
  346. speechsynth.text = sentence;
  347. window.speechSynthesis.speak(speechsynth);
  348. }
  349. function testSpeechOut () {
  350. console.log('click');
  351. speechsynth.text = 'test 123';
  352. speechsynth.volume = 1;
  353. speechsynth.rate = 1;
  354. console.log(speechsynth);
  355. window.speechSynthesis.speak(speechsynth);
  356. console.log(window.speechSynthesis);
  357. }
  358. function recognizeSpeech () {
  359. // if (state === 'answer') {
  360. // var arr;
  361. // switch (question) {
  362. // case 1:
  363. // arr = QUESTION_ONE_QUESTIONS;
  364. // break;
  365. // case 2:
  366. // // arr = QUESTION_TWO_QUESTIONS;
  367. // break;
  368. // case 3:
  369. // arr = [1, 2, 3, 4, 5, 6, 7, 8, 9];
  370. // break;
  371. // case 4:
  372. // break;
  373. // case 5:
  374. // break;
  375. // }
  376. // // var grammar = '#JSGF V1.0; grammar colors; public <color> = ' + arr.join(' | ') + ' ;';
  377. // // var speechRecognitionList = new SpeechGrammarList();
  378. // // speechRecognitionList.addFromString(grammar, 1);
  379. // // recognition.grammars = speechRecognitionList;
  380. // }
  381. recognition.start();
  382. console.log('reocgnition started. Question: ' + question);
  383. recognition.onresult = function (event) {
  384. var last = event.results.length - 1;
  385. var speechResult = event.results[last][0].transcript.toLowerCase();
  386. diagnosticPara.textContent += speechResult + ' ';
  387. // console.log('Confidence: ' + event.results[0][0].confidence)
  388. console.log('process: ' + speechResult);
  389. processSpeech(speechResult);
  390. // testBtn.disabled = false
  391. // testBtn.textContent = 'record...'
  392. };
  393. function processSpeech (speechResult) {
  394. console.log('To dialogflow: ' + speechResult);
  395. ws.send(speechResult);
  396. let timeOut;
  397. switch (question) {
  398. case 1:
  399. timeOut = 6500;
  400. break;
  401. case 2:
  402. answerQuery += speechResult;
  403. return;
  404. case 3:
  405. if (speechResult.includes('uhr')) {
  406. speechResult = speechResult.replace('uhr', '');
  407. }
  408. timeOut = 6500;
  409. break;
  410. case 4:
  411. break;
  412. case 5:
  413. timeOut = 6500;
  414. break;
  415. }
  416. if (state === 'answer') {
  417. if (timerId != undefined) {
  418. clearTimeout(timerId);
  419. }
  420. answerQuery += speechResult;
  421. timerId = window.setTimeout(
  422. function () {
  423. // if (!rePrompt) {
  424. // ws.send('ich brauche noch etwas Zeit')
  425. // } else {
  426. console.log('recording end. Evaluate: ' + answerQuery);
  427. handleAnswer(answerQuery);
  428. answerQuery = '';
  429. diagnosticPara.textContent = '';
  430. // }
  431. recognition.stop();
  432. console.log('timer fallback');
  433. }, timeOut);
  434. } else {
  435. console.log('recording end.');
  436. recognition.stop();
  437. }
  438. }
  439. recognition.onspeechend = function () {
  440. // recognition.stop();
  441. // testBtn.disabled = false;
  442. // testBtn.textContent = 'Start new test';
  443. };
  444. recognition.onerror = function (event) {
  445. testBtn.disabled = false;
  446. testBtn.textContent = 'Start new test';
  447. diagnosticPara.textContent = 'Error occurred in recognition: ' + event.error;
  448. };
  449. recognition.onaudiostart = function (event) {
  450. // Fired when the user agent has started to capture audio.
  451. };
  452. recognition.onaudioend = function (event) {
  453. };
  454. recognition.onend = function (event) {
  455. // Fired when the speech recognition service has disconnected.
  456. };
  457. recognition.onnomatch = function (event) {
  458. // Fired when the speech recognition service returns a final result with no significant recognition. This may involve some degree of recognition, which doesn't meet or exceed the confidence threshold.
  459. // console.log('SpeechRecognition.onnomatch')
  460. };
  461. recognition.onsoundstart = function (event) {
  462. // Fired when any sound — recognisable speech or not — has been detected.
  463. };
  464. recognition.onsoundend = function (event) {
  465. // Fired when any sound — recognisable speech or not — has stopped being detected.
  466. };
  467. recognition.onspeechstart = function (event) {
  468. // Fired when sound that is recognised by the speech recognition service as speech has been detected.
  469. };
  470. recognition.onstart = function (event) {
  471. // Fired when the speech recognition service has begun listening to incoming audio with intent to recognize grammars associated with the current SpeechRecognition.
  472. };
  473. }
  474. function calculatePoints (tokens, dict) {
  475. let points = 0;
  476. for (let word of tokens) {
  477. if (dict[word] !== undefined) {
  478. points += dict[word];
  479. }
  480. }
  481. return points;
  482. }
  483. // #endregion
  484. testBtn.addEventListener('click', startDemenzScreening);
  485. testBtn2.addEventListener('click', testSpeechOut);