DemenzTestBachelorArbeit/client/ws-client.js

// #region web speech recognition api
var SpeechRecognition = SpeechRecognition || webkitSpeechRecognition;
var SpeechGrammarList = SpeechGrammarList || webkitSpeechGrammarList;
var SpeechRecognitionEvent = SpeechRecognitionEvent || webkitSpeechRecognitionEvent;
// #endregion

// #region state management
var state = '';
var question = 0;
// currently disabled feature
// var rePrompt = false;
var partTwo = false;
var questionThreeCount = 0;
var strike = 0;
// #endregion

// #region  questions
const QUESTION_ONE = 'Ich werde Ihnen jetzt langsam eine Liste mit Worten vorlesen. Danach wiederholen Sie bitte möglichst viele dieser Worte. Auf die Reihenfolge kommt es nicht an.';
const QUESTION_ONE_PT2 = 'Vielen Dank. Nun nenne ich Ihnen die gleichen 10 Worte ein zweites mal. Auch danach sollen Sie wieder möglichst viele Worte wiederholen';
const QUESTION_TWO = 'Nennen Sie mir bitte so viel Dinge wie möglich, die man im Supermarkt kaufen kann. Sie haben dafür eine Minute Zeit. Und Los';
const QUESTION_THREE = 'Ich werde Ihnen jetzt eine Zahlenreihe nennen, die Sie mir dann bitte in umgekehrter Reihenfolge wiederholen sollen. Wenn ich beispielsweise, vier - fünf sage, dann sagen Sie bitte, fünf - vier.';
const QUESTION_FOUR = 'Wir kommen nun zur vierten Frage. Bitte zählen Sie von 76 an rückwärts. Nennen Sie die Zahlen abwechselnd als ganze Zahlen und als einzelne Ziffern. Zum Beispiel:';
const QUESTION_FOUR_PT2 = ['34', '3, 3', '32', '3, 1'];
const QUESTION_FIVE = 'Ganz am Anfang dieses Tests habe ich Ihnen 10 Worte genannt. Können Sie sich noch an diese Worte erinnern?';
// #endregion

const FINISH = 'Sie sind jetzt fertig mit dem Test, Vielen Dank.';

// #region intents
const WELCOME_INTENT = 'Default Welcome Intent';
const WELCOME_FOLLOWUP_YES = 'Default Welcome Intent - yes';
const WELCOME_FOLLOWUP_NO = 'Default Welcome Intent - no';
// currently disabled feature
// const MORE_TIME = 'Add Time Intent';
// const MORE_TIME_YES = 'Add Time Intent - yes';
// const MORE_TIME_NO = 'Add Time Intent - no';
const QUIT_INTENT = 'Quit Intent';
const FALLBACK_INTENT = 'Default Fallback Intent';
const HELP_INTENT = 'Help Intent';
const CHOOSE_QUESTION = 'Frage_Starten';
// disabled feature
// const NEXT_QUESTION = 'Nächste Frage';
// #endregion

// #region questions and expected results
const QUESTION_ONE_ANSWERS = { 'teller': 1, 'hund': 1, 'lampe': 1, 'brief': 1, 'apfel': 1, 'apfelwiese': 2, 'apfelbaum': 2, 'und': 1, 'hose': 1, 'tisch': 1, 'wiese': 1, 'glas': 1, 'baum': 1 };
const QUESTION_ONE_QUESTIONS = ['teller', 'hund', 'lampe', 'brief', 'apfel', 'hose', 'tisch', 'wiese', 'glas', 'baum'];
const QUESTION_TWO_ANSWERS = {};
var QUESTION_TWO_QUESTIONS;
const QUESTION_THREE_QUESTIONS_PT1 = ['7, 2', '4, 7, 9', '5, 4, 9, 6', '2, 7, 5, 3, 6', '8, 1, 3, 5, 4, 2'];
const QUESTION_THREE_QUESTIONS_PT2 = ['8, 6', '3, 1, 5', '1, 9, 7, 4', '1, 3, 5, 4, 8', '4, 1, 2, 7, 9, 5'];
const QUESTION_THREE_ANSWERS_PT1 = ['27', '974', '6945', '63572', '245318'];
const QUESTION_THREE_ANSWERS_PT2 = ['68', '513', '4791', '84531', '597214'];
LoadQuestionTwo();
function LoadQuestionTwo () {
  var xmlhttp;
  if (window.XMLHttpRequest) { // code for IE7+, Firefox, Chrome, Opera, Safari
    xmlhttp = new XMLHttpRequest();
  } else { // code for IE6, IE5
    xmlhttp = new ActiveXObject('Microsoft.XMLHTTP');
  }
  xmlhttp.onreadystatechange = function () {
    if (xmlhttp.readyState === 4 && xmlhttp.status === 200) {
      var text = xmlhttp.responseText.toLowerCase();
      // Now convert it into array using regex
      QUESTION_TWO_QUESTIONS = text.split('\r\n');
      for (let word of QUESTION_TWO_QUESTIONS) {
        QUESTION_TWO_ANSWERS[word] = 1;
      }
    }
  };
  xmlhttp.open('GET', 'lebensmittel.txt', true);
  xmlhttp.send();
}
// #endregion

// #region points
const questionPoints = {
  1: 0,
  2: 0,
  3: 0,
  4: 0,
  5: 0 };
  // #endregion

// tokenization
const separators = [' ', '\\\+', '-', '\\\(', '\\\)', '\\*', '/', ':', '\\\?'];

// Timers
var timerId;

// #region html elements
var serverPara = document.querySelector('.server');
var diagnosticPara = document.querySelector('.output');
var testBtn = document.querySelector('button');
var testBtn2 = document.getElementById('speechBtn');
var infoPara = document.getElementById('info');
var questionNumDisplay = document.querySelector('.quest');
// #endregion

// websocket to communicate with the server
var ws = new WebSocket('wss://' + window.location.host + window.location.pathname + 'ws');

// #region speech recognition initialization
var recognition = new SpeechRecognition();
recognition.lang = 'de-DE';
// recognition.interimResults = false;
recognition.maxAlternatives = 1;
recognition.continuous = true;
var answerQuery = '';
var proceedWithRecording = false;
// #endregion

// #region speech synthesis initialization
var speechsynth = new SpeechSynthesisUtterance();
var listSpeechsynth = new SpeechSynthesisUtterance();
var voices;
// #endregion

// #region speech events
window.speechSynthesis.onvoiceschanged = function () {
  voices = window.speechSynthesis.getVoices();
  voices.forEach(element => {
    if (element.name === 'Google Deutsch') {
      speechsynth.voice = element;
      listSpeechsynth.voice = element;
    }
  });
  speechsynth.rate = 0.88;
  listSpeechsynth.rate = 0.7;
};

speechsynth.onend = function (event) {
  if (proceedWithRecording) {
    recognition.start();
    console.log('reocgnition started. Question: ' + question);
  }
  proceedWithRecording = false;
  diagnosticPara.textContent = '';
  console.log('global speech end');
};
// #endregion

// #region websocket events
ws.onopen = function () {
  serverPara.style.background = 'green';
  serverPara.innerHTML = 'Server online';
};

ws.onmessage = function (payload) {
  var dialogflowResult = JSON.parse(payload.data);
  checkIntent(dialogflowResult);
};
// #endregion

// INTENT HANDLING
function checkIntent (result) {
  switch (result.intent.displayName) {
    case QUIT_INTENT:
      state = 'quit';
      stopRecognitionFallback();
      speak('Beende die Durchführung.');
      break;
    case WELCOME_INTENT:
      state = 'detect';
      proceedWithRecording = true;
      speak(result.fulfillmentText);
      break;
    case WELCOME_FOLLOWUP_YES:
      startQuestion(1);
      break;
    case WELCOME_FOLLOWUP_NO:
      speak('Alles klar. Danke fürs Benutzen.');
      break;
    /// /// /// currently disabled feature
    // case MORE_TIME:
    //   state = 'detect';
    //   proceedWithRecording = true;
    //   speak('Brauchen Sie noch etwas Zeit?');
    //   break;
    // case MORE_TIME_YES:
    //   rePrompt = true;
    //   state = 'answer';
    //   proceedWithRecording = true;
    //   speak('Alles klar');
    //   break;
    // case MORE_TIME_NO:
    //   state = 'answer';
    //   speak('Verstanden');
    //   recognition.stop();
    //   ws.send(answerQuery);
    //   break;
    // case CHOOSE_QUESTION:
    //     question = result.parameters.fields.num.numberValue;
    //     state = 'answer';
    //     handleQuestion();
    //     break;
    /// /// ///
    case HELP_INTENT:
      stopRecognitionFallback();
      recognition.stop();
      startQuestion(question);
      break;
    case FALLBACK_INTENT:
      // fallback what should happen if no intent is detected
      // this is handled in the processSpeech() function
      break;
    default:
      break;
  }
}

// #region question handling
function startQuestion (number) {
  question = number;
  state = 'answer';
  answerQuery = '';
  questionNumDisplay.textContent = 'Question: ' + question;
  handleQuestion();
}

function handleQuestion () {
  switch (question) {
    case 1:
      readQuestionOne(QUESTION_ONE);
      break;
    case 2:
      readQuestionTwo();
      break;
    case 3:
      readQuestionThree();
      break;
    case 4:
      readQuestionFour();
      break;
    case 5:
      proceedWithRecording = true;
      speak(QUESTION_FIVE);
      break;
  }
}

function readQuestionOne (questionText) {
  speak(questionText);
  for (let i = 0; i < QUESTION_ONE_QUESTIONS.length; i++) {
    let utterance = new SpeechSynthesisUtterance();
    utterance.voice = voices[2];
    utterance.rate = 0.75;
    utterance.text = QUESTION_ONE_QUESTIONS[i];
    window.speechSynthesis.speak(utterance);
    if (i === 9) {
      utterance.onend = function (event) {
        recognition.start();
        console.log('reocgnition started. Question: ' + question);
      };
    }
  }
}

function readQuestionTwo () {
  let utterance = new SpeechSynthesisUtterance();
  utterance.voice = voices[2];
  utterance.text = QUESTION_TWO;
  utterance.rate = 0.8;
  window.speechSynthesis.speak(utterance);
  utterance.onend = function (event) {
    window.setTimeout(
      function () {
        recognition.stop();
        window.setTimeout(
          function () {
            handleAnswer(answerQuery);
            answerQuery = '';
          }, 3000);
      }, 6000);
    recognition.start();
    console.log('reocgnition started. Question: ' + question);
  };
}

function readQuestionThree () {
  speak('Dankeschön, weiter geht es mit der nächsten Frage.');
  let utterance = new SpeechSynthesisUtterance();
  utterance.voice = voices[2];
  utterance.text = QUESTION_THREE;
  utterance.rate = 0.88;
  window.speechSynthesis.speak(utterance);
  utterance.onend = function (event) {
    proceedWithRecording = true;
    speak(QUESTION_THREE_QUESTIONS_PT1[questionThreeCount]);
  };
}

function readQuestionFour () {
  speak(QUESTION_FOUR);
  for (let i = 0; i < QUESTION_FOUR_PT2.length; i++) {
    let utterance = new SpeechSynthesisUtterance();
    utterance.voice = voices[2];
    utterance.rate = 0.75;
    utterance.text = QUESTION_FOUR_PT2[i];
    window.speechSynthesis.speak(utterance);
  }
  speak('Sie haben hierfür 1 Minute Zeit.');
}

function handleAnswer (query) {
  switch (question) {
    case 1:
      handleAnswerToFirstQuestion(query);
      break;
    case 2:
      handleAnswerToSecondQuestion(query);
      break;
    case 3:
      handleAnswerToThirdQuestion(query);
      break;
    case 4:
      // not possible with the current state of web speech api
      // handleAnswerToFourthQuestion(query);
      break;
    case 5:
      handleAnswerToFifthQuestion(query);
      break;
  }
}

function handleAnswerToFirstQuestion (answer) {
  deconstructAndCalculateAnswer(answer);
  if (partTwo) {
    partTwo = false;
    speak('Vielen Dank, nun geht es weiter mit der nächsten Frage');
    startQuestion(2);
  } else {
    // currently disabled feature
    // rePrompt = false;
    speak(QUESTION_ONE_PT2);
    readQuestionOne(QUESTION_ONE_PT2);
    partTwo = true;
  }
}

function handleAnswerToSecondQuestion (answer) {
  deconstructAndCalculateAnswer(answer);
  startQuestion(3);
}

function handleAnswerToThirdQuestion (query) {
  speechsynth.rate = 0.88;
  query = query.replace(' ', '');

  let answerArray;
  let questionArray;
  if (!partTwo) {
    answerArray = QUESTION_THREE_ANSWERS_PT1;
  } else {
    answerArray = QUESTION_THREE_ANSWERS_PT2;
  }
  if (query === answerArray[questionThreeCount]) {
    strike = 0;
    partTwo = false;
    questionThreeCount++;
    questionPoints[question] = questionThreeCount + 1;
    questionArray = QUESTION_THREE_QUESTIONS_PT1;
  } else {
    strike++;
    partTwo = true;
    questionArray = QUESTION_THREE_QUESTIONS_PT2;
  }

  if (strike === 2 || questionThreeCount === 5) {
    speechsynth.rate = 0.88;
    console.log('question 3 points: ' + questionPoints[question]);
    speak('weiter geht es mit der Nächsten Frage');
    // Question 4 is not possible to take, because of technical limitation
    // startQuestion(4);

    startQuestion(5);
    return;
  }
  proceedWithRecording = true;
  speak(questionArray[questionThreeCount]);

  console.log('count: ' + questionThreeCount + ', strike: ' + strike + ', points: ' + questionPoints[question]);
}

// not possible with the current state of web speech api
// function handleAnswerToFourthQuestion (answer) {
//   // In order to evaluate the answer, the recognizer has to distinguish between the number 77 spoken "seventy seven" and 7-7 spoken "seven seven", which it doesnt
//   startQuestion(5);
// }

function handleAnswerToFifthQuestion (answer) {
  deconstructAndCalculateAnswer(answer);
  speak(FINISH);
}

// #endregion

// NOT SUPPORTED
// function recognizeSpeech () {
//   var arr;
//   switch (question) {
//     case 1:
//       arr = QUESTION_ONE_QUESTIONS;
//       break;
//     case 2:
//       arr = QUESTION_TWO_QUESTIONS;
//       break;
//     case 3:
//       arr = [1, 2, 3, 4, 5, 6, 7, 8, 9];
//       break;
//     case 4:
//       arr = ['eins', 'zwei', 'drei', 'vier', 'fünf', 'sechs', 'sieben', 'acht', 'neun'];
//       break;
//     case 5:
//       break;
//   }

//   var grammar = '#JSGF V1.0; grammar arr; public <arr> = ' + arr.join(' | ') + ' ;';
//   var speechRecognitionList = new SpeechGrammarList();
//   speechRecognitionList.addFromString(grammar, 1);
//   recognition.grammars = speechRecognitionList;

//   recognition.start();
// }

// #region speech recognition event
recognition.onresult = function (event) {
  var last = event.results.length - 1;
  var speechResult = event.results[last][0].transcript.toLowerCase();

  diagnosticPara.textContent += speechResult + ' ';

  // console.log('Confidence: ' + event.results[0][0].confidence)
  console.log('process: ' + speechResult);
  processSpeech(speechResult);
  // testBtn.disabled = false
  // testBtn.textContent = 'record...'
};

recognition.onspeechend = function () {
  // recognition.stop();
  // testBtn.disabled = false;
  // testBtn.textContent = 'Start new test';
};

recognition.onerror = function (event) {
  testBtn.disabled = false;
  testBtn.textContent = 'Start new test';
  diagnosticPara.textContent = 'Error occurred in recognition: ' + event.error;
};

recognition.onaudiostart = function (event) {
  // Fired when the user agent has started to capture audio.

};

recognition.onaudioend = function (event) {

};

recognition.onend = function (event) {
  // Fired when the speech recognition service has disconnected.

};

recognition.onnomatch = function (event) {
  // Fired when the speech recognition service returns a final result with no significant recognition. This may involve some degree of recognition, which doesn't meet or exceed the confidence threshold.
  // console.log('SpeechRecognition.onnomatch')
};

recognition.onsoundstart = function (event) {
  // Fired when any sound — recognisable speech or not — has been detected.

};

recognition.onsoundend = function (event) {
  // Fired when any sound — recognisable speech or not — has stopped being detected.

};

recognition.onspeechstart = function (event) {
  // Fired when sound that is recognised by the speech recognition service as speech has been detected.

};
recognition.onstart = function (event) {
  // Fired when the speech recognition service has begun listening to incoming audio with intent to recognize grammars associated with the current SpeechRecognition.

};
// }
// #endregion

// #region global functions
function processSpeech (speechResult) {
  console.log('To dialogflow: ' + speechResult);
  ws.send(speechResult);

  let timeOut;
  switch (question) {
    case 1:
      timeOut = 6500;
      break;
    case 2:
      answerQuery += speechResult;
      return;
    case 3:
      if (speechResult.includes('uhr')) {
        speechResult = speechResult.replace('uhr', '');
      }
      timeOut = 6500;
      break;
    case 4:
      break;
    case 5:
      timeOut = 6500;
      break;
  }

  if (state === 'answer') {
    stopRecognitionFallback();
    answerQuery += speechResult;
    timerId = window.setTimeout(
      function () {
        // currently disabled
        // if (!rePrompt) {
        //   ws.send('ich brauche noch etwas Zeit')
        // } else {
        console.log('recording end. Evaluate: ' + answerQuery);
        handleAnswer(answerQuery);
        diagnosticPara.textContent = '';
        // }
        recognition.stop();
        console.log('timer fallback');
      }, timeOut);
  } else {
    console.log('recording end.');
    recognition.stop();
  }
}

function startDemenzScreening () {
  // ws.send('starte demenz test');
  startQuestion(4);
  testBtn.disabled = true;
  testBtn.textContent = 'Test in progress';
  infoPara.textContent = 'wait...';
  diagnosticPara.textContent = 'detecting...';
}

function testSpeechOut () {
  question = 4;
}

function speak (sentence) {
  speechsynth.text = sentence;
  window.speechSynthesis.speak(speechsynth);
}

function stopRecognitionFallback () {
  if (timerId != undefined) {
    clearTimeout(timerId);
  }
}

function deconstructAndCalculateAnswer (answer) {
  let questionAnswers;
  switch (question) {
    case 1:
      questionAnswers = QUESTION_ONE_ANSWERS;
      break;
    case 2:
      questionAnswers = QUESTION_TWO_ANSWERS;
      break;
    case 5:
      questionAnswers = QUESTION_ONE_ANSWERS;
      break;
  }
  var tokens = answer.split(new RegExp(separators.join('|'), 'g'));
  questionPoints[question] += calculatePoints(tokens, questionAnswers);
}

function calculatePoints (tokens, d) {
  let points = 0;
  let dict = {};
  Object.assign(dict, d);
  for (let word of tokens) {
    if (dict[word] !== undefined) {
      points += dict[word];
      delete dict[word];
    }
  }
  return points;
}
// #endregion

testBtn.addEventListener('click', startDemenzScreening);
testBtn2.addEventListener('click', testSpeechOut);