// todo: felolvasás kiszűrése var recognition = null; var wavRecorder = null; var wavRecorder_send = true; var selected_language = null; var utterance_selected_language = null; var utterance_selected_voice = null; var send_voice_message_to_server = true; function auto_grow(element) { element.style.height = (element.scrollHeight) + "px"; } function chat_keypress(e){ if (e.key === "Enter" && !e.shiftKey) { e.preventDefault(); // e.currentTarget.closest("form").submit(); send_message(); } } function chat_oninput(){ console.log("oninput...") } function send_message(audio_file=null){ message = document.getElementById('chat_msg_input').value if (message.length > 0){ karc.href(this, '/self_service_pos/ajax/chat_message.json', data={'msg': message, 'audio_file': audio_file}); document.getElementById("chat_msg_input").value = null; document.getElementById("chat_msg_input").setAttribute("rows", "1"); document.getElementById("chat_msg_input").style.height = null; } } function chat_message_arrived(message, from_self, html=null){ console.log("message arrived: ") console.log(message) const messagesDiv = document.getElementById('chat_messages'); messagesDiv.insertAdjacentHTML('beforeend', html); messagesDiv.scrollTop = messagesDiv.scrollHeight; if (!from_self){ speakText(message.translated_text || message.text, message.target_language) } } async function speakText(text, lang = "hu") { console.log("speak text: "+text+" - lang: "+lang); // leállítja a felismerést if (wavRecorder != null){ // await wavRecorder.pause(); wavRecorder_send = false; } // leállítja a jelenlegi beszédet window.speechSynthesis.cancel(); // hangok betöltése function loadVoices() { return new Promise(function(resolve) { let voices = window.speechSynthesis.getVoices(); if (voices.length) { resolve(voices); return; } window.speechSynthesis.onvoiceschanged = function() { voices = window.speechSynthesis.getVoices(); resolve(voices); }; }); } console.log("utterance selected lang: "+utterance_selected_language) console.log("utterance selected voice: "+utterance_selected_voice) if (utterance_selected_language == null || !utterance_selected_language.startsWith(lang)){ voices = await loadVoices(); utterance_selected_voice = null; utterance_selected_language = "hu-HU"; for (let voice of voices) { if (voice.lang && voice.lang.startsWith(lang)) { utterance_selected_voice = voice; utterance_selected_language = voice.lang; break; } } } // felolvasás console.log("speak text language: "+utterance_selected_language); const utterance = new SpeechSynthesisUtterance(text); utterance.lang = utterance_selected_language; if (utterance_selected_voice) utterance.voice = utterance_selected_voice; utterance.onerror = function(event){ console.error("Felolvasási hiba:", event.error, event); } utterance.onend = function(event){ console.log("speak end"); // elindítja a felismerést if (wavRecorder != null){ wavRecorder_send = null; } } utterance.onstart = function(event){ console.log("speak start"); } window.speechSynthesis.speak(utterance); } function playAudio(audio_file){ console.log("play audio file") var audio = new Audio("/self_service_pos/chat/audio/"+audio_file); audio.play(); } //#region Recognition function startRecognition(language){ console.log("start recognition") send_voice_message_to_server = true if (recognition == null){ // recognition = new window.SpeechRecognition() recognition = window.SpeechRecognition ? new window.SpeechRecognition() : new window.webkitSpeechRecognition() recognition.continuous = true recognition.interimResults = true recognition.lang = language // Set the results handler recognition.onresult = (event) => { let interimTranscript = '' let finalTranscript = null for (let i = event.resultIndex; i < event.results.length; ++i) { // In Chrome, result is final when speech stops for ~500ms-1s if (event.results[i].isFinal) { finalTranscript = event.results[i][0].transcript console.log(finalTranscript) // Do something with the final result... } else { console.log(event.results[i][0].transcript) interimTranscript += event.results[i][0].transcript // Interim result // For example: each spoken word or pair of words // Do something with the interim result... } } document.getElementById("chat_msg_input").value = (finalTranscript != null) ? finalTranscript : interimTranscript; if(finalTranscript != null && finalTranscript.length > 0){ if (send_voice_message_to_server){ send_message() } } } recognition.onerror = (error) => { console.error("Recognition error: ", error) } recognition.onspeechend = function() { console.log('Beszéd vége.'); }; recognition.onnomatch = function() { console.log('Nem sikerült felismerni a beszédet.'); }; } recognition.start() } function stopRecognition(){ console.log("Stop recognition") try{ result = recognition.stop(); console.log(result); } catch(err){ console.error("Stop recognition error:", err); } } //#endregion function newChatSTTConnection(force_connect=false){ console.log("New chat STT connection - force: "+force_connect+", chat_ws: "+chat_ws); //connecting if (chat_ws == null || force_connect){ chat_ws = new WebSocket('WSS://'+window.location.hostname + ':' + window.location.port+'/self_service_pos/chat_stt_websocket'); console.log(chat_ws); chat_ws.onopen = function(event){ console.log("chat_ws open"); chat_ws.send(JSON.stringify({"language": document.getElementById("chat_container").attributes['data-selected_language'].value})) } chat_ws.onclose = async function(event){ console.log("chat_ws close"); // setTimeout(()=>{ // newChatSTTConnection(true); // }, 1000) } chat_ws.onmessage = async (res) =>{ console.log("chat_ws onmessage") console.log(res) if (res.data.includes("transcript")){ textarea = document.getElementById("chat_msg_input"); textarea.value = JSON.parse(res.data).transcript; auto_grow(textarea); } else if (res.data.includes("send_message")){ if(send_voice_message_to_server){ send_message(JSON.parse(res.data).audio_file); } } } chat_ws.onerror = function(error){ console.error("chat_ws error: ", error); } } } //#region Recognition full audio var mediaRecorder = null; var google_stream = null; var chat_ws = null; async function startRecognitionServer(){ console.log("Start recognition python") newChatSTTConnection(); if (mediaRecorder && mediaRecorder.state === "recording"){ mediaRecorder.stop(); } else{ audioChunks = []; google_stream = await navigator.mediaDevices.getUserMedia({ audio: true }); mediaRecorder = new MediaRecorder(google_stream, { mimeType: "audio/webm" }); mediaRecorder.ondataavailable = (event) => { console.log("mediarecorder on data available") if(event.data.size > 0 && chat_ws.readyState === 1){ event.data.arrayBuffer().then(buf => { chat_ws.send(buf) }) } } mediaRecorder.onstop = async () => { console.log("mediarecorder on stop") if(google_stream){ google_stream.getTracks().forEach((track) => track.stop()); delete google_stream } } mediaRecorder.start(); console.log("Mediarecorder start") } } function stopRecognitionServer(){ mediaRecorder.stop(); } //#endregion //#region Recognition streaming async function startStreamingRecognition(){ newChatSTTConnection(); try{ if(wavRecorder != null){ await stopStreamingRecognition(); } wavRecorder = new WavRecorder({sampleRate: 24000}) console.log(wavRecorder.getStatus()); await wavRecorder.begin(); console.log(wavRecorder.getStatus()); await wavRecorder.record((data) => { // console.log("record, send: "+wavRecorder_send) if(wavRecorder_send == true){ chat_ws.send(data.mono); } else if(wavRecorder_send == null){ console.log("last not sent - utterance") wavRecorder_send = true; } }) chat_ws.send(JSON.stringify({"language": document.getElementById("chat_container").attributes['data-selected_language'].value})) chat_ws.send(JSON.stringify({"streaming_thread_open":true})) wavRecorder_send = true; $(".streaming_recognition_on").prop("hidden", false); $(".streaming_recognition_off").prop("hidden", true); } catch(e){ console.error(e); stopStreamingRecognition() } } async function stopStreamingRecognition(){ if(chat_ws != null){ chat_ws.send(JSON.stringify({"streaming_thread_close":true})) } if(wavRecorder != null){ try{ try{ await wavRecorder.pause(); console.log(wavRecorder.getStatus()); } catch(e){ console.log(e) } const finalAudio = await wavRecorder.end(); console.log("final audio") console.log(finalAudio); wavRecorder.listenForDeviceChange((deviceList) => {}); } catch(e){ console.error(e); try{ wavRecorder.stream.getTracks().forEach((track) => track.stop()); } catch(e){} } } wavRecorder = null; $(".streaming_recognition_off").prop("hidden", false); $(".streaming_recognition_on").prop("hidden", true); } //#endregion