import { useEffect, useRef, useState } from "react"; import { TbBrandOpenai, TbMicrophone2, TbPlayerPlay, TbPlayerStop, } from "react-icons/tb"; import "./App.css"; type ChatMsg = { role: string; content: string; }; let audioBlobs: Array = []; let streamBeingCaptured: MediaStream | null = null; let mediaRecorder: MediaRecorder | null = null; function get_mic() { if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) { console.log("getUserMedia supported."); return navigator.mediaDevices.getUserMedia({ audio: true }); } throw "getUserMedia not supported on your browser!"; } function startRecord() { audioBlobs = []; get_mic().then((stream) => { console.log("got mic"); streamBeingCaptured = stream; mediaRecorder = new MediaRecorder(stream); console.log("Starting Recording"); mediaRecorder.addEventListener("dataavailable", (event) => { audioBlobs.push(event.data); }); mediaRecorder.start(); }); } function stopRecord() { if (!mediaRecorder) { throw "MediaRecorder not set"; } if (!streamBeingCaptured) { throw "Stream not set"; } mediaRecorder.stop(); streamBeingCaptured.getTracks() .forEach((track) => track.stop()); mediaRecorder = null; streamBeingCaptured = null; console.log("Starting Recording"); console.log(audioBlobs); } function playRecord() { const audioBlob = new Blob(audioBlobs, { type: "audio/webm" }); const audioUrl = URL.createObjectURL(audioBlob); const audio = new Audio(audioUrl); audio.play(); } function playMsg(msg: ChatMsg) { const audio = new Audio( "http://100.82.51.22:8001/speak?" + new URLSearchParams({ text: msg.content }), ); console.log("loading audio and playing?"); audio.play(); } function Header() { return (
Speach to Speech AI example
); } function Feed(props: { chat: Array; setChatStateFn: any }) { const bottomRef = useRef(null); const scrollToBottom = () => { if (bottomRef.current) { bottomRef.current.scrollIntoView({ behavior: "smooth" }); } }; useEffect(() => { scrollToBottom(); console.log("scroll?"); }); return (
{props.chat.filter((m: ChatMsg) => m.role != "system").map(( m: ChatMsg, i: number, ) => )}
); } function Msg(props: { msg: ChatMsg }) { return (
{props.msg.role.toUpperCase()}:
{props.msg.content}
); } function Controls(props: { setChatStateFn: any, chat: Array }) { const [recordState, setRecordState] = useState(false); function toggleRecord() { if (recordState == false) { startRecord(); setRecordState(true); } else { stopRecord(); setRecordState(false); } } function sendAudio() { var formData = new FormData(); formData.append("audio", new Blob(audioBlobs, { type: "audio/webm" })); fetch("http://100.82.51.22:8001/get-text", { "method": "POST", "body": formData, }).then((res) => res.json()) .then((res) => { props.setChatStateFn((curState: Array) => [ ...curState, { "role": "user", "content": res["user-transcript"] }, ]); fetch("http://100.82.51.22:8001/conversation", { "method": "POST", "body": JSON.stringify([...props.chat, { "role": "user", "content": res["user-transcript"], }]), }).then((res) => res.json()) .then((res) => { props.setChatStateFn(( curState: Array, ) => [...curState, res]); console.log("attempting to play result"); playMsg(res); }); }); } return (
); } function App() { const [chatState, setChatState] = useState([{ role: "system", content: "You are a helpful assistant.", }]); return ( <>

); } export default App;