diff --git "a/ML/\354\242\214\355\221\234\352\260\222 \353\260\233\354\225\204\354\204\234 \354\266\234\353\240\245.py" "b/ML/\354\242\214\355\221\234\352\260\222 \353\260\233\354\225\204\354\204\234 \354\266\234\353\240\245.py" index e59f038..4bfd6a9 100644 --- "a/ML/\354\242\214\355\221\234\352\260\222 \353\260\233\354\225\204\354\204\234 \354\266\234\353\240\245.py" +++ "b/ML/\354\242\214\355\221\234\352\260\222 \353\260\233\354\225\204\354\204\234 \354\266\234\353\240\245.py" @@ -21,9 +21,9 @@ async def handle_client(websocket, path): # 클라이언트로부터 메시지를 기다립니다. message = await websocket.recv() print(message) - data = json.loads(message) - if len(data) == 0: + if len(message) == 0 or message == "undefined": continue + data = [json.loads(message)] numpy_array = np.array([[item["x"], item["y"], item["z"]] for item in data[0]], dtype=np.float32) num_joints = 21 @@ -70,7 +70,7 @@ async def handle_client(websocket, path): # WebSocket 서버를 시작합니다. -start_server = websockets.serve(handle_client, "localhost", 8080) # IP 주소와 포트를 설정합니다. +start_server = websockets.serve(handle_client, "localhost", 8081) # IP 주소와 포트를 설정합니다. async def main(): diff --git a/front/src/pages/TranslatePage/translate/Input.tsx b/front/src/pages/TranslatePage/translate/Input.tsx index b9120e5..a609a0a 100644 --- a/front/src/pages/TranslatePage/translate/Input.tsx +++ b/front/src/pages/TranslatePage/translate/Input.tsx @@ -23,18 +23,53 @@ const Input = () => { const handleUserMedia = () => setTimeout(() => setLoading(false), 1_000); const [text, setText] = useRecoilState(resultText); + const [isChecked, setIsChecked] = useState(false); + + const handleCheckboxChange = () => { + setIsChecked(!isChecked); + setText(""); + }; + + /* 랜드마크들의 좌표를 콘솔에 출력 및 websocket으로 전달 */ + const OutputData = useCallback(() => { + if (webcamRef.current !== null) { + const results = resultsRef.current!; + if (resultsRef.current) { + console.log(results.rightHandLandmarks); + // 웹소켓으로 데이터 전송 + if (socketRef_hands.current.readyState === WebSocket.OPEN) { + socketRef_hands.current.send( + JSON.stringify(results.rightHandLandmarks) + ); + console.log("hands sended"); + } else { + console.error("ws connection is not open"); + } + } + } + }, [webcamRef]); + const capture = useCallback(() => { const imageSrc = webcamRef.current?.getScreenshot(); const imgData: string | undefined = imageSrc?.toString()?.substr(23); - if (imgData) { - socket.send(imgData); + if (imgData && !isChecked) { + if (socketRef.current.readyState === WebSocket.OPEN) { + socketRef.current.send(imgData); + } else { + console.error("ws connection is not open. (8080)"); + } // console.log(imgData); } }, [webcamRef]); useEffect(() => { - const interval = setInterval(capture, 33.33); - return () => clearInterval(interval); - }, [capture]); + if (!isChecked) { + const interval = setInterval(capture, 1000 / 30); + return () => clearInterval(interval); + } else { + const interval = setInterval(OutputData, 1000); + return () => clearInterval(interval); + } + }, [capture, webcamRef, OutputData, isChecked]); /** * 검출결과（프레임마다 호출됨） @@ -127,85 +162,159 @@ const Input = () => { } }, [onPoseResults]); - const socket = new WebSocket("ws://localhost:8080"); - socket.onmessage = (event) => { + const socketRef = useRef(new WebSocket("ws://localhost:8080")); + const socketRef_hands = useRef( + new WebSocket("ws://localhost:8081") + ); + + // useEffect(() => { + // if (isChecked) { + // if (socketRef.current.readyState === WebSocket.OPEN) { + // socketRef.current.close(); + // console.log("disconnected"); + // const socket = new WebSocket("ws://localhost:8081"); + // socketRef.current = socket; // 지문자 모드 + // console.log("changed"); + // } + // } else { + // if (socketRef.current.readyState === WebSocket.OPEN) { + // socketRef.current.close(); + // console.log("disconnected"); + // const socket = new WebSocket("ws://localhost:8080"); + // socketRef.current = socket; + // console.log("changed"); + // } + // } + + // return () => { + // socketRef.current.onclose = () => {}; // 연결 종료 메시지를 방지하기 위해 빈 함수를 설정합니다. + // socketRef.current.close(); + // }; + // }, [isChecked]); + + socketRef.current.onmessage = (event) => { + console.log(`receive message: ${event.data}`); + const jsonString = JSON.parse(event.data); + if (isChecked) { + setText(text + jsonString.result); + } else { + setText(jsonString.result); + } + console.log(text); + }; + socketRef_hands.current.onmessage = (event) => { console.log(`receive message: ${event.data}`); const jsonString = JSON.parse(event.data); - setText(text + jsonString.result); + if (isChecked) { + setText(text + jsonString.result); + } else { + setText(jsonString.result); + } console.log(text); }; useEffect(() => { - socket.onopen = () => { + socketRef.current.onopen = () => { console.log("ws connected"); }; - socket.onclose = () => { + socketRef.current.onclose = () => { console.log("ws closed"); }; - socket.onmessage = (event) => { + socketRef.current.onmessage = (event) => { console.log(`receive message: ${event.data}`); + const jsonString = JSON.parse(event.data); + if (isChecked) { + setText(text + jsonString.result); + } else { + setText(jsonString.result); + } + + console.log(text); }; return () => { - socket.close(); + socketRef.current.close(); }; }, []); - /* 랜드마크들의 좌표를 콘솔에 출력 및 websocket으로 전달 */ - // const OutputData = () => { - // if (!loading) { - // if (webcamRef.current !== null) { - // const results = resultsRef.current!; - // if (resultsRef.current) { - // console.log(results.multiHandLandmarks); - // // 웹소켓으로 데이터 전송 - // if (socket.readyState === WebSocket.OPEN) { - // socket.send(JSON.stringify(results.multiHandLandmarks)); - // } else { - // console.error("ws connection is not open"); - // } - // } - // } - // } - // }; - useEffect(() => { - const intervalId = setInterval(capture, 33.33); + socketRef_hands.current.onopen = () => { + console.log("ws connected"); + }; + socketRef_hands.current.onclose = () => { + console.log("ws closed"); + }; + socketRef_hands.current.onmessage = (event) => { + console.log(`receive message: ${event.data}`); + const jsonString = JSON.parse(event.data); + if (isChecked) { + setText(text + jsonString.result); + } else { + setText(jsonString.result); + } + + console.log(text); + }; return () => { - clearInterval(intervalId); + socketRef_hands.current.close(); }; - }, [capture]); + }, []); return (
{loading && ( -
+
로딩 중...
)} -
+ +
{/* 비디오 캡쳐 */} -
- -
+ + {/* 랜드마크를 손에 표시 */}
diff --git a/front/src/pages/TranslatePage/translate/NotTranslating.tsx b/front/src/pages/TranslatePage/translate/NotTranslating.tsx index fe17d90..347bc42 100644 --- a/front/src/pages/TranslatePage/translate/NotTranslating.tsx +++ b/front/src/pages/TranslatePage/translate/NotTranslating.tsx @@ -2,7 +2,7 @@ import { BsPersonFill } from "react-icons/bs"; export const NotTranslating = () => { return ( -
+

diff --git a/front/src/pages/TranslatePage/translate/Translate.tsx b/front/src/pages/TranslatePage/translate/Translate.tsx index acbb2e0..df8df23 100644 --- a/front/src/pages/TranslatePage/translate/Translate.tsx +++ b/front/src/pages/TranslatePage/translate/Translate.tsx @@ -85,7 +85,7 @@ const Translate = () => { {openModal && }