Merge pull request #29 from DEVOCEAN-YOUNG-404/seungjun

feat: 지문자/수어 모드 변경
DEVOCEAN-YOUNG-404 · Nov 1, 2023 · 158ae45 · 158ae45
2 parents 15ea187 + 5f70c5e
commit 158ae45
Show file tree

Hide file tree

Showing 4 changed files with 167 additions and 58 deletions.
diff --git a/ML/좌표값 받아서 출력.py b/ML/좌표값 받아서 출력.py
@@ -21,9 +21,9 @@ async def handle_client(websocket, path):
             # 클라이언트로부터 메시지를 기다립니다.
             message = await websocket.recv()
             print(message)
-            data = json.loads(message)
-            if len(data) == 0:
+            if len(message) == 0 or message == "undefined":
                 continue
+            data = [json.loads(message)]
             numpy_array = np.array([[item["x"], item["y"], item["z"]] for item in data[0]], dtype=np.float32)
 
             num_joints = 21
@@ -70,7 +70,7 @@ async def handle_client(websocket, path):
 
 
 # WebSocket 서버를 시작합니다.
-start_server = websockets.serve(handle_client, "localhost", 8080)  # IP 주소와 포트를 설정합니다.
+start_server = websockets.serve(handle_client, "localhost", 8081)  # IP 주소와 포트를 설정합니다.
 
 
 async def main():

diff --git a/front/src/pages/TranslatePage/translate/Input.tsx b/front/src/pages/TranslatePage/translate/Input.tsx
@@ -23,18 +23,53 @@ const Input = () => {
   const handleUserMedia = () => setTimeout(() => setLoading(false), 1_000);
   const [text, setText] = useRecoilState(resultText);
 
+  const [isChecked, setIsChecked] = useState(false);
+
+  const handleCheckboxChange = () => {
+    setIsChecked(!isChecked);
+    setText("");
+  };
+
+  /*  랜드마크들의 좌표를 콘솔에 출력 및 websocket으로 전달 */
+  const OutputData = useCallback(() => {
+    if (webcamRef.current !== null) {
+      const results = resultsRef.current!;
+      if (resultsRef.current) {
+        console.log(results.rightHandLandmarks);
+        // 웹소켓으로 데이터 전송
+        if (socketRef_hands.current.readyState === WebSocket.OPEN) {
+          socketRef_hands.current.send(
+            JSON.stringify(results.rightHandLandmarks)
+          );
+          console.log("hands sended");
+        } else {
+          console.error("ws connection is not open");
+        }
+      }
+    }
+  }, [webcamRef]);
+
   const capture = useCallback(() => {
     const imageSrc = webcamRef.current?.getScreenshot();
     const imgData: string | undefined = imageSrc?.toString()?.substr(23);
-    if (imgData) {
-      socket.send(imgData);
+    if (imgData && !isChecked) {
+      if (socketRef.current.readyState === WebSocket.OPEN) {
+        socketRef.current.send(imgData);
+      } else {
+        console.error("ws connection is not open. (8080)");
+      }
       // console.log(imgData);
     }
   }, [webcamRef]);
   useEffect(() => {
-    const interval = setInterval(capture, 33.33);
-    return () => clearInterval(interval);
-  }, [capture]);
+    if (!isChecked) {
+      const interval = setInterval(capture, 1000 / 30);
+      return () => clearInterval(interval);
+    } else {
+      const interval = setInterval(OutputData, 1000);
+      return () => clearInterval(interval);
+    }
+  }, [capture, webcamRef, OutputData, isChecked]);
 
   /**
    * 검출결과（프레임마다 호출됨）
@@ -127,85 +162,159 @@ const Input = () => {
     }
   }, [onPoseResults]);
 
-  const socket = new WebSocket("ws://localhost:8080");
-  socket.onmessage = (event) => {
+  const socketRef = useRef<WebSocket>(new WebSocket("ws://localhost:8080"));
+  const socketRef_hands = useRef<WebSocket>(
+    new WebSocket("ws://localhost:8081")
+  );
+
+  // useEffect(() => {
+  //   if (isChecked) {
+  //     if (socketRef.current.readyState === WebSocket.OPEN) {
+  //       socketRef.current.close();
+  //       console.log("disconnected");
+  //       const socket = new WebSocket("ws://localhost:8081");
+  //       socketRef.current = socket; // 지문자 모드
+  //       console.log("changed");
+  //     }
+  //   } else {
+  //     if (socketRef.current.readyState === WebSocket.OPEN) {
+  //       socketRef.current.close();
+  //       console.log("disconnected");
+  //       const socket = new WebSocket("ws://localhost:8080");
+  //       socketRef.current = socket;
+  //       console.log("changed");
+  //     }
+  //   }
+
+  //   return () => {
+  //     socketRef.current.onclose = () => {}; // 연결 종료 메시지를 방지하기 위해 빈 함수를 설정합니다.
+  //     socketRef.current.close();
+  //   };
+  // }, [isChecked]);
+
+  socketRef.current.onmessage = (event) => {
+    console.log(`receive message: ${event.data}`);
+    const jsonString = JSON.parse(event.data);
+    if (isChecked) {
+      setText(text + jsonString.result);
+    } else {
+      setText(jsonString.result);
+    }
+    console.log(text);
+  };
+  socketRef_hands.current.onmessage = (event) => {
     console.log(`receive message: ${event.data}`);
     const jsonString = JSON.parse(event.data);
-    setText(text + jsonString.result);
+    if (isChecked) {
+      setText(text + jsonString.result);
+    } else {
+      setText(jsonString.result);
+    }
     console.log(text);
   };
 
   useEffect(() => {
-    socket.onopen = () => {
+    socketRef.current.onopen = () => {
       console.log("ws connected");
     };
-    socket.onclose = () => {
+    socketRef.current.onclose = () => {
       console.log("ws closed");
     };
-    socket.onmessage = (event) => {
+    socketRef.current.onmessage = (event) => {
       console.log(`receive message: ${event.data}`);
+      const jsonString = JSON.parse(event.data);
+      if (isChecked) {
+        setText(text + jsonString.result);
+      } else {
+        setText(jsonString.result);
+      }
+
+      console.log(text);
     };
     return () => {
-      socket.close();
+      socketRef.current.close();
     };
   }, []);
 
-  /*  랜드마크들의 좌표를 콘솔에 출력 및 websocket으로 전달 */
-  // const OutputData = () => {
-  //   if (!loading) {
-  //     if (webcamRef.current !== null) {
-  //       const results = resultsRef.current!;
-  //       if (resultsRef.current) {
-  //         console.log(results.multiHandLandmarks);
-  //         // 웹소켓으로 데이터 전송
-  //         if (socket.readyState === WebSocket.OPEN) {
-  //           socket.send(JSON.stringify(results.multiHandLandmarks));
-  //         } else {
-  //           console.error("ws connection is not open");
-  //         }
-  //       }
-  //     }
-  //   }
-  // };
-
   useEffect(() => {
-    const intervalId = setInterval(capture, 33.33);
+    socketRef_hands.current.onopen = () => {
+      console.log("ws connected");
+    };
+    socketRef_hands.current.onclose = () => {
+      console.log("ws closed");
+    };
+    socketRef_hands.current.onmessage = (event) => {
+      console.log(`receive message: ${event.data}`);
+      const jsonString = JSON.parse(event.data);
+      if (isChecked) {
+        setText(text + jsonString.result);
+      } else {
+        setText(jsonString.result);
+      }
+
+      console.log(text);
+    };
     return () => {
-      clearInterval(intervalId);
+      socketRef_hands.current.close();
     };
-  }, [capture]);
+  }, []);
 
   return (
     <div className="">
       {loading && (
-        <div className="z-10 absolute w-[20rem] h-[20rem] md:w-[25rem] xl:w-[31.25rem] md:h-[31.25rem] xl:h-[37.5rem] rounded-xl border border-gray-200 shadow-md flex items-center justify-center">
+        <div className="z-10 absolute w-[20rem] h-[20rem] md:w-[25rem] xl:w-[37.5rem] md:h-[31.25rem] xl:h-[37.5rem] rounded-xl border border-gray-200 shadow-md flex items-center justify-center">
           로딩 중...
         </div>
       )}
-      <div className="relative w-[20rem] h-[20rem] md:w-[25rem] xl:w-[31.25rem] md:h-[31.25rem] xl:h-[37.5rem] overflow-hidden flex flex-col items-center justify-center rounded-[15px]">
+      <label className="absolute z-10 inline-flex items-center mt-2 ml-2 cursor-pointer select-none themeSwitcherTwo">
+        <input
+          type="checkbox"
+          checked={isChecked}
+          onChange={handleCheckboxChange}
+          className="sr-only"
+        />
+        <span className="flex items-center text-sm font-medium text-black text-white font-main label">
+          지문자 모드
+        </span>
+        <span
+          className={`slider mx-4 flex h-8 w-[60px] items-center rounded-full p-1 duration-200 ${
+            isChecked ? "bg-main-2" : "bg-[#CCCCCE]"
+          }`}
+        >
+          <span
+            className={`dot h-6 w-6 rounded-full bg-white duration-200 ${
+              isChecked ? "translate-x-[28px]" : ""
+            }`}
+          ></span>
+        </span>
+      </label>
+      <div className="-scale-x-100 relative w-[20rem] h-[20rem] md:w-[25rem] xl:w-[37.5rem] md:h-[31.25rem] xl:h-[37.5rem] overflow-hidden flex flex-col items-center justify-center rounded-[15px]">
         {/* 비디오 캡쳐 */}
-        <div className="w-full">
-          <Webcam
-            audio={false}
-            style={{
-              visibility: loading ? "hidden" : "visible",
-              objectFit: "cover",
-            }}
-            width={1280}
-            height={960}
-            ref={webcamRef}
-            onUserMedia={handleUserMedia}
-            screenshotFormat="image/jpeg"
-            videoConstraints={{ width: 1280, height: 960, facingMode: "user" }}
-          />
-        </div>
+
+        <Webcam
+          audio={false}
+          style={{
+            visibility: loading ? "hidden" : "visible",
+            objectFit: "fill",
+            position: "absolute",
+            width: "100%",
+            height: "100%",
+          }}
+          mirrored={false}
+          width={600}
+          height={600}
+          ref={webcamRef}
+          onUserMedia={handleUserMedia}
+          screenshotFormat="image/jpeg"
+          videoConstraints={{ width: 600, height: 600, facingMode: "user" }}
+        />
 
         {/* 랜드마크를 손에 표시 */}
         <canvas
           ref={canvasPoseRef}
-          className="absolute w-[20rem] h-[20rem] md:w-[25rem] xl:w-[31.25rem] md:h-[31.25rem] xl:h-[37.5rem] bg-white"
-          width={1280}
-          height={960}
+          className="absolute w-[20rem] h-[20rem] md:w-[25rem] xl:w-[37.5rem] md:h-[31.25rem] xl:h-[37.5rem] bg-white"
+          width={600}
+          height={600}
         />
       </div>
     </div>

diff --git a/front/src/pages/TranslatePage/translate/NotTranslating.tsx b/front/src/pages/TranslatePage/translate/NotTranslating.tsx
@@ -2,7 +2,7 @@ import { BsPersonFill } from "react-icons/bs";
 
 export const NotTranslating = () => {
   return (
-    <div className="flex flex-col items-center justify-center w-[20rem] h-[20rem] md:w-[25rem] xl:w-[31.25rem] md:h-[31.25rem] xl:h-[37.5rem] bg-gray-200 border-none rounded-xl">
+    <div className="flex flex-col items-center justify-center w-[20rem] h-[20rem] md:w-[25rem] xl:w-[37.5rem] md:h-[31.25rem] xl:h-[37.5rem] bg-gray-200 border-none rounded-xl">
       <p className="text-[250px] md:text-[400px] text-gray-300">
         <BsPersonFill />
       </p>

diff --git a/front/src/pages/TranslatePage/translate/Translate.tsx b/front/src/pages/TranslatePage/translate/Translate.tsx
@@ -85,7 +85,7 @@ const Translate = () => {
       {openModal && <ConfigModal onOpenModal={onModalAlert} />}
       <button
         onClick={onModalAlert}
-        className="w-[8rem] h-[2.5rem] md:w-[160px] md:h-[3rem] font-main text-xl font-bold items-end justify-end ml-[12rem] md:ml-[48.7rem] xl:ml-[61rem] text-white bg-main-2 rounded-lg"
+        className="w-[8rem] h-[2.5rem] md:w-[160px] md:h-[3rem] font-main text-xl font-bold items-end justify-end ml-[12rem] md:ml-[48.7rem] xl:ml-[67.5rem] text-white bg-main-2 rounded-lg"
       >
         연동 설정
       </button>