DemenzTestBachelorArbeit/client/ws-client.js

// web speech recognition api
var SpeechRecognition = SpeechRecognition || webkitSpeechRecognition;
var SpeechGrammarList = SpeechGrammarList || webkitSpeechGrammarList;
var SpeechRecognitionEvent = SpeechRecognitionEvent || webkitSpeechRecognitionEvent;

// state management
var state = '';
var question = 0;
var rePrompt = false;
var partTwo = false;
var count = 0;
var strike = 0;

// Questions
const QUESTION_ONE = 'Ich werde Ihnen jetzt langsam eine Liste mit Worten vorlesen. Danach wiederholen Sie bitte möglichst viele dieser Worte. Auf die Reihenfolge kommt es nicht an.';
const QUESTION_ONE_PT2 = 'Vielen Dank. Nun nenne ich Ihnen die gleichen 10 Worte ein zweites mal. Auch danach sollen Sie wieder möglichst viele Worte wiederholen';
const QUESTION_TWO = 'Nennen Sie mir bitte so viel Dinge wie möglich, die man im Supermarkt kaufen kann. Sie haben dafür eine Minute Zeit.';
const QUESTION_THREE = 'Ich werde Ihnen jetzt eine Zahlenreihe nennen, die Sie mir dann bitte in umgekehrter Reihenfolge wiederholen sollen. Wenn ich beispielsweise, vier - fünf sage, dann sagen Sie bitte, fünf - vier.';

// intents
const WELCOME_INTENT = 'Default Welcome Intent';
const WELCOME_FOLLOWUP_YES = 'Default Welcome Intent - yes';
const WELCOME_FOLLOWUP_NO = 'Default Welcome Intent - no';
const MORE_TIME = 'Add Time Intent';
const MORE_TIME_YES = 'Add Time Intent - yes';
const MORE_TIME_NO = 'Add Time Intent - no';
const QUIT_INTENT = 'Quit Intent';
const FALLBACK_INTENT = 'Default Fallback Intent';
const HELP_INTENT = 'Help Intent';
const CHOOSE_QUESTION = 'Frage_Starten';
const NEXT_QUESTION = 'Nächste Frage';

// question one results
const QUESTION_ONE_ANSWERS = ['teller', 'hund', 'lampe', 'brief', 'apfel', 'apfelwiese', 'apfelwiese', 'apfelbaum', 'apfelbaum', 'und', 'hose', 'tisch', 'wiese', 'glas', 'baum'];
const QUESTION_ONE_QUESTIONS = ['teller', 'hund', 'lampe', 'brief', 'apfel', 'hose', 'tisch', 'wiese', 'glas', 'baum'];
const QUESTION_TWO_ANSWERS = [''];
const QUESTION_TWO_QUESTIONS = [''];
const QUESTION_THREE_QUESTIONS_PT1 = ['7, 2', '4, 7, 9', '5, 4, 9, 6', '2, 7, 5, 3, 6', '8, 1, 3, 5, 4, 2'];
const QUESTION_THREE_QUESTIONS_PT2 = ['8, 6', '3, 1, 5', '1, 9, 7, 4', '1, 3, 5, 4, 8', '4, 1, 2, 7, 9, 5'];
const QUESTION_THREE_ANSWERS_PT1 = ['27', '974', '6945', '63572', '245318'];
const QUESTION_THREE_ANSWERS_PT2 = ['68', '513', '4791', '84531', '597214'];

// points
var questionOnePoints = 0;
var questionTwoPoints = 0;
var questionThreePoints = 0;
var questionFourPoints = 0;
var questionFivePoints = 0;
var questionSixPoints = 0;

// tokenization
const separators = [' ', '\\\+', '-', '\\\(', '\\\)', '\\*', '/', ':', '\\\?'];

// Timers
var timerId;

// html elements
var serverPara = document.querySelector('.server');
var diagnosticPara = document.querySelector('.output');
var testBtn = document.querySelector('button');
var testBtn2 = document.getElementById('speechBtn');
var infoPara = document.getElementById('info');
var userPrompt = document.getElementById('query');

console.log(window.location.host + window.location.pathname);

// websocket to communicate with the server
var ws = new WebSocket('ws://localhost:8000/ws');

// speech recognition
var recognition = new SpeechRecognition();
recognition.lang = 'de-DE';
// recognition.interimResults = false;
recognition.maxAlternatives = 1;
recognition.continuous = true;
var answerQuery = '';
var skipRecording = false;

// speech synthesis
var speechsynth = new SpeechSynthesisUtterance();
var listSpeechsynth = new SpeechSynthesisUtterance();
var voices;

window.speechSynthesis.onvoiceschanged = function () {
  voices = window.speechSynthesis.getVoices();
  voices.forEach(element => {
    if (element.name === 'Google Deutsch') {
      speechsynth.voice = element;
      listSpeechsynth.voice = element;
    }
  });

  listSpeechsynth.rate = 0.7;
};

function startDemenzScreening () {
  console.log('button clicked');
  ws.send('starte demenz test');
  //
  // state = 'answer'
  // recognizeSpeech()

  //
  testBtn.disabled = true;
  testBtn.textContent = 'Test in progress';
  infoPara.textContent = 'wait...';
}

function speak (sentence) {
  speechsynth.text = sentence;
  window.speechSynthesis.speak(speechsynth);
}

function testSpeechOut () {
  console.log('click');
  // skipRecording = true
  // let utterance = new SpeechSynthesisUtterance()
  // utterance.voice = voices[2]
  // // utterance.rate = 0.75
  // utterance.text = QUESTION_THREE
  // window.speechSynthesis.speak(utterance)
  //
  // question = 3;
  // startQuestion(question);
  // state = 'answer';
  speechsynth.text = 'test 123';
  speechsynth.volume = 1;
  speechsynth.rate = 1;
  console.log(speechsynth);
  window.speechSynthesis.speak(speechsynth);
  console.log(window.speechSynthesis);
  // speak('Niko ist der größte hurn eu west');
}

// websocket events
ws.onopen = function () {
  serverPara.style.background = 'green';
  serverPara.innerHTML = 'Server online';
};

ws.onmessage = function (payload) {
  var dialogflowResult = JSON.parse(payload.data);
  checkIntent(dialogflowResult);
  document.querySelector('h1').innerHTML = dialogflowResult.intent.displayName;
};

// INTENT HANDLING
function checkIntent (result) {
  switch (result.intent.displayName) {
    case QUIT_INTENT:
      state = 'quit';
      skipRecording = true;
      speak('Okay, Danke fürs Benutzen.');
      break;
    case WELCOME_INTENT:
      state = 'detect';
      // speak(result.fulfillmentText)
      speak('go?');
      break;
    case WELCOME_FOLLOWUP_YES:
      startQuestion(1);
      break;
    case WELCOME_FOLLOWUP_NO:
      skipRecording = true;
      speak('Okay, Danke fürs Benutzen.');
      break;
    case MORE_TIME:
      state = 'detect';
      speak('Brauchen Sie noch etwas Zeit?');
      break;
    case MORE_TIME_YES:
      rePrompt = true;
      state = 'answer';
      speak('Alles klar');
      break;
    case MORE_TIME_NO:
      skipRecording = true;
      state = 'answer';
      speak('Verstanden');
      recognition.stop();
      ws.send(answerQuery);
      break;
    case CHOOSE_QUESTION:
      question = result.parameters.fields.num.numberValue;
      state = 'answer';
      handleQuestion();
      break;
    case FALLBACK_INTENT:
      // if (state === 'answer') {
      //   handleAnswer(result.queryText)
      // }
      break;
    default:
      break;
  }
}

function startQuestion (number) {
  question = number;
  state = 'answer';
  handleQuestion();
}

// QUESTION HANDLING
function handleQuestion () {
  switch (question) {
    case 1:
      readQuestionOne(QUESTION_ONE);
      break;
    case 2:
      readQuestionTwo(QUESTION_TWO);
      break;
    case 3:
      // skipRecording = true;
      // speak(QUESTION_THREE);
      speak(QUESTION_THREE_QUESTIONS_PT1[0]);
      break;
    case 4:
      break;
    case 5:
      break;
  }
}

function handleAnswer (query) {
  switch (question) {
    case 1:
      handleAnswerToFirstQuestion(query);
      break;
    case 2:
      handleAnswerToSecondQuestion(query);
      break;
    case 3:
      handleAnswerToThirdQuestion(query);
      break;
    case 4:
      break;
    case 5:
      break;
  }
}

function handleAnswerToFirstQuestion (answer) {
  var tokens = answer.split(new RegExp(separators.join('|'), 'g'));
  questionOnePoints = calculatePoints(tokens, QUESTION_ONE_ANSWERS);
  if (partTwo) {
    partTwo = false;
    startQuestion(2);
    // state = 'detect'
  } else {
    rePrompt = false;
    speak(QUESTION_ONE_PT2);
    partTwo = true;
  }
}

function handleAnswerToSecondQuestion (answer) {
  var tokens = answer.split(new RegExp(separators.join('|'), 'g'));
  questionTwoPoints = calculatePoints(tokens, QUESTION_TWO_ANSWERS);
  skipRecording = true;
  speak('Sie haben ' + questionOnePoints + 'Punkte');
  startQuestion(3);
  // state = 'detect'
}

function handleAnswerToThirdQuestion (query) {
  speechsynth.rate = 0.87;
  query = query.replace(' ', '');

  let answerArray;
  let questionArray;
  if (!partTwo) {
    answerArray = QUESTION_THREE_ANSWERS_PT1;
  } else {
    answerArray = QUESTION_THREE_ANSWERS_PT2;
  }
  if (query === answerArray[count]) {
    strike = 0;
    partTwo = false;
    count++;
    questionThreePoints = count + 1;
    questionArray = QUESTION_THREE_QUESTIONS_PT1;
  } else {
    strike++;
    partTwo = true;
    questionArray = QUESTION_THREE_QUESTIONS_PT2;
  }

  if (strike === 2 || count === 5) {
    speechsynth.rate = 1;
    skipRecording = true;
    speak('weiter geht es mit der Nächsten Frage');
    startQuestion(4);
    return;
  }

  speak(questionArray[count]);

  console.log('count: ' + count + ', strike: ' + strike + ', points: ' + questionThreePoints);
}

// Question specific functions
function readQuestionOne (text) {
  skipRecording = true;
  speak(text);
  for (let i = 0; i < QUESTION_ONE_QUESTIONS.length; i++) {
    let utterance = new SpeechSynthesisUtterance();
    utterance.voice = voices[2];
    utterance.rate = 0.75;
    utterance.text = QUESTION_ONE_QUESTIONS[i];
    window.speechSynthesis.speak(utterance);
    if (i === 9) {
      utterance.onend = function (event) {
        console.log('end of aufzählung' + i);
        recognizeSpeech();
      };
    }
  }
}

function readQuestionTwo (text) {
  let utterance = new SpeechSynthesisUtterance();
  utterance.voice = voices[2];
  utterance.text = text;
  window.speechSynthesis.speak(utterance);
  console.log('q 2 started');
  utterance.onend = function (event) {
    window.setTimeout(
      function () {
        recognition.stop();
        handleAnswer(answerQuery);
      }, 60000);
    console.log('q 2 recognition started');
    recognizeSpeech();
  };
}

function calculatePoints (tokens, array) {
  let points = 0;
  for (let i of array) {
    for (let j of tokens) {
      if (i === j) {
        points++;
      }
    }
  }
  return points;
}

speechsynth.onend = function (event) {
  switch (question) {
    case 1:
      break;
    case 2:
      break;
    case 3:
      break;
    case 4:
      break;
    case 5:
      break;
  }
  if (!skipRecording) {
    recognizeSpeech();
  }
  skipRecording = false;
  console.log('global speech end');
};

function recognizeSpeech () {
  if (state === 'answer') {
    var arr;
    switch (question) {
      case 1:
        arr = QUESTION_ONE_QUESTIONS;
        break;
      case 2:
        return;
      case 3:
        arr = [1, 2, 3, 4, 5, 6, 7, 8, 9];
        break;
      case 4:
        break;
      case 5:
        break;
    }

    var grammar = '#JSGF V1.0; grammar colors; public <color> = ' + arr.join(' | ') + ' ;';
    var speechRecognitionList = new SpeechGrammarList();
    speechRecognitionList.addFromString(grammar, 1);
    recognition.grammars = speechRecognitionList;
  }
  recognition.start();
  console.log('reocgnition started');

  recognition.onresult = function (event) {
    var last = event.results.length - 1;
    var speechResult = event.results[last][0].transcript.toLowerCase();

    diagnosticPara.textContent += speechResult + ' ';

    // console.log('Confidence: ' + event.results[0][0].confidence)

    processSpeech(speechResult);
    // testBtn.disabled = false
    // testBtn.textContent = 'record...'
  };

  function processSpeech (speechResult) {
    console.log('To dialogflow: ' + speechResult);
    ws.send(speechResult);

    let timeOut;
    switch (question) {
      case 1:
        timeOut = 6500;
        break;
      case 2:
        answerQuery += speechResult;
        return;
      case 3:
        if (speechResult.includes('uhr')) {
          speechResult = speechResult.replace('uhr', '');
        }
        timeOut = 6500;
        break;
      case 4:
        break;
      case 5:
        break;
    }

    if (state === 'answer') {
      if (timerId != undefined) {
        clearTimeout(timerId);
      }
      answerQuery += speechResult;
      timerId = window.setTimeout(
        function () {
          // if (!rePrompt) {
          //   ws.send('ich brauche noch etwas Zeit')
          // } else {
          console.log('recording end. Evaluate: ' + answerQuery);
          handleAnswer(answerQuery);
          answerQuery = '';
          diagnosticPara.textContent = '';
          // }
          recognition.stop();
          console.log('timer fallback');
        }, timeOut);
    } else {
      console.log('recording end.');
      recognition.stop();
    }
  }

  recognition.onspeechend = function () {
    // recognition.stop();
    // testBtn.disabled = false;
    // testBtn.textContent = 'Start new test';
  };

  recognition.onerror = function (event) {
    testBtn.disabled = false;
    testBtn.textContent = 'Start new test';
    diagnosticPara.textContent = 'Error occurred in recognition: ' + event.error;
  };

  recognition.onaudiostart = function (event) {
    // Fired when the user agent has started to capture audio.

  };

  recognition.onaudioend = function (event) {

  };

  recognition.onend = function (event) {
    // Fired when the speech recognition service has disconnected.

  };

  recognition.onnomatch = function (event) {
    // Fired when the speech recognition service returns a final result with no significant recognition. This may involve some degree of recognition, which doesn't meet or exceed the confidence threshold.
    // console.log('SpeechRecognition.onnomatch')
  };

  recognition.onsoundstart = function (event) {
    // Fired when any sound — recognisable speech or not — has been detected.

  };

  recognition.onsoundend = function (event) {
    // Fired when any sound — recognisable speech or not — has stopped being detected.

  };

  recognition.onspeechstart = function (event) {
    // Fired when sound that is recognised by the speech recognition service as speech has been detected.

  };
  recognition.onstart = function (event) {
    // Fired when the speech recognition service has begun listening to incoming audio with intent to recognize grammars associated with the current SpeechRecognition.

  };
}

testBtn.addEventListener('click', startDemenzScreening);
testBtn2.addEventListener('click', testSpeechOut);