Staging

.github/workflows/main.yml

@@ -34,7 +34,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -e .[tfcpu,api,dev,testing,linting]
+          pip install -e .[tfcpu,api,dev,testing,linting,game]
 
       - name: Precommit check
         run: |

.github/workflows/release.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -e .[tfcpu,api,dev,testing,linting]
+          pip install -e .[tfcpu,api,dev,testing,linting,game]
 
       - name: Test with pytest
         run: |
@@ -30,18 +30,17 @@ jobs:
       - name: Build
         run: |
           pip install setuptools sdist wheel twine
-          pip install -e .[tfcpu,api]
+          pip install -e .[tfcpu,api,game]
           python setup.py sdist bdist_wheel
-
+
+      # - name: Publish distribution 📦 to PyPI
+      #   uses: pypa/gh-action-pypi-publish@master
+      #   with:
+      #     password: ${{ secrets.PYPI_API_TOKEN }}
       - uses: "marvinpinto/action-automatic-releases@latest"
         with:
           repo_token: "${{ secrets.GITHUB_TOKEN }}"
           prerelease: false
           files: |
             ./dist/*tar.gz
             ./dist/*.whl
-
-      - name: Publish distribution 📦 to PyPI
-        uses: pypa/gh-action-pypi-publish@master
-        with:
-          password: ${{ secrets.PYPI_API_TOKEN }}

README.md

@@ -4,6 +4,10 @@ This repo contains a basic procedure to train and deploy the DNN model suggested
 Network Architectures from the paper, <br>
 <img src="resources/dfpmodel.png" width="50%"><img src="resources/features.png" width="50%">
 
+
+### Additional feature (pygame)
+![TF2DeepFloorplan_3dviz](resources/raycast.gif)
+
 ## Requirements
 Depends on different applications, the following installation methods can
 
@@ -13,9 +17,10 @@ Depends on different applications, the following installation methods can
 |Ubuntu|GPU|Model Development|`pip install -e .[tfgpu,dev,testing,linting]`|
 |MacOS|M1 Chip|Model Development|`pip install -e .[tfmacm1,dev,testing,linting]`|
 |Ubuntu|GPU|Model Deployment API|`pip install -e .[tfgpu,api]`|
-|Ubuntu|GPU|Model Development and Deployment API|`pip install -e .[tfgpu,api,dev,testing,linting]`|
+|Ubuntu|GPU|Everything|`pip install -e .[tfgpu,api,dev,testing,linting,game]`|
 |Agnostic|...|Docker|(to be updated)|
 |Ubuntu|GPU|Notebook|`pip install -e .[tfgpu,jupyter]`|
+|Ubuntu|GPU|Game|`pip install -e .[tfgpu,game]`|
 
 ## How to run?
 1. Install packages.
@@ -84,6 +89,10 @@ python -m dfp.deploy [--image 'path/to/image']
 python -m dfp.deploy --image floorplan.jpg --weight log/store/G
 --postprocess --colorize --save output.jpg --loadmethod log
 ```
+8. Play with pygame.
+```
+python -m dfp.game
+```
 
 ## Docker for API
 1. Build and run docker container. (Please train your weight, google drive does not work currently due to its update.)

docs/game.toml

@@ -0,0 +1,13 @@
+tfmodel = 'subclass'
+# image = 'resources/123.jpg'
+# image = 'resources/example4.png'
+image = 'resources/example5.jpg'
+CASTED_RAYS = 150
+postprocess = 1
+colorize = 0
+save = ''
+weight = 'model/store_vgg16_exp2_model.tflite'
+loadmethod = 'tflite'
+feature_channels = [32, 32, 32, 32]
+backbone = 'vgg16'
+feature_names = ["block1_pool", "block2_pool", "block3_pool", "block4_pool", "block5_pool",]

setup.cfg

@@ -55,6 +55,9 @@ linting =
 jupyter =
     jupyterlab
     notebook
+game =
+    pygame
+    numba
 
 [tool:pytest]
 testpaths =

src/dfp/game/__main__.py

@@ -0,0 +1,261 @@
+import logging
+import math
+import random
+import sys
+import time
+from argparse import Namespace
+from typing import Dict, Tuple, Union
+
+import numpy as np
+import pygame
+from numba import jit, njit
+
+from ..deploy import main
+from ..utils.settings import overwrite_args_with_toml
+
+logging.basicConfig(level=logging.INFO)
+
+args = Namespace(tomlfile="docs/game.toml")
+args = overwrite_args_with_toml(args)
+FOV = math.pi / 3
+HALF_FOV = FOV / 2
+STEP_ANGLE = FOV / args.casted_rays
+
+
+def draw_dfp(map: np.ndarray, win: pygame.Surface):
+    surf = pygame.surfarray.make_surface(bg)
+    win.blit(surf, (0, 0))
+
+
+def show_fps(win: pygame.Surface, clock: pygame.time.Clock):
+    font = pygame.font.SysFont("Arial", 18)
+    fps = str(int(clock.get_fps()))
+    fps_text = font.render(fps, 1, pygame.Color("coral"))
+    win.blit(fps_text, (10, 0))
+
+
+def player_control(
+    keys: Dict[int, bool],
+    angle: float,
+    x: float,
+    y: float,
+    w: int,
+    h: int,
+    binary_map: np.ndarray,
+) -> Tuple[float, float, float]:
+    dangle = 0.01
+    dpos = 1
+    origx, origy = x, y
+    if keys[pygame.K_LEFT]:
+        angle -= dangle
+    if keys[pygame.K_RIGHT]:
+        angle += dangle
+    if keys[pygame.K_w]:
+        x -= math.sin(angle) * dpos
+        y += math.cos(angle) * dpos
+    if keys[pygame.K_s]:
+        x += math.sin(angle) * dpos
+        y -= math.cos(angle) * dpos
+    if keys[pygame.K_d]:
+        x += math.sin(angle - math.pi / 2) * dpos
+        y -= math.cos(angle - math.pi / 2) * dpos
+    if keys[pygame.K_a]:
+        x -= math.sin(angle - math.pi / 2) * dpos
+        y += math.cos(angle - math.pi / 2) * dpos
+    if keys[pygame.K_q]:
+        pygame.quit()
+        sys.exit(0)
+    if x < 0:
+        x = 0
+    if x >= w:
+        x = w - 1
+    if y < 0:
+        y = 0
+    if y >= h:
+        y = h - 1
+    if binary_map[int(y), int(x)] == 1:
+        return angle, origx, origy
+    return angle, x, y
+
+
+def draw_player_direction(
+    x: float,
+    y: float,
+    angle: float,
+    depth: Union[float, int],
+    win: pygame.Surface,
+):
+    pygame.draw.line(
+        win,
+        (0, 255, 0),
+        (x, y),
+        (
+            x - math.sin(angle) * depth,
+            y + math.cos(angle) * depth,
+        ),
+        3,
+    )
+
+
+@njit
+def ray_cast_dda(
+    x: float,
+    y: float,
+    angle: float,
+    depth: Union[float, int],
+    w: int,
+    h: int,
+    binary_map: np.ndarray,
+) -> Tuple[float, float, bool]:
+    vPlayer = np.array([x, y])
+    vRayStart = vPlayer.copy()
+    vMouseCell = np.array(
+        [x - math.sin(angle) * depth, y + math.cos(angle) * depth]
+    )
+    vRayDir = (vMouseCell - vPlayer) / np.linalg.norm((vMouseCell - vPlayer))
+    vRayUnitStepSize = np.array(
+        [
+            np.sqrt(1 + (vRayDir[1] / (vRayDir[0] + 1e-10)) ** 2),
+            np.sqrt(1 + (vRayDir[0] / (vRayDir[1] + 1e-10)) ** 2),
+        ]
+    )
+    vMapCheck = vRayStart.copy()
+    vRayLength1D = np.array([0.0, 0.0])
+    vStep = np.array([0, 0])
+
+    for i in range(2):
+        vStep[i] = -1 if vRayDir[i] < 0 else 1
+        # vRayLength1D[i] = vRayUnitStepSize[i]
+        # vRayLength1D[i] = (
+        #     (vRayStart[i] - vMapCheck[i]) * vRayUnitStepSize[i]
+        #     if vRayDir[i] < 0
+        #     else ((vMapCheck[i] + 1) - vRayStart[i]) * vRayUnitStepSize[i]
+        # )
+
+    bTileFound = False
+    fMaxDistance = depth
+    fDistance = 0
+    while not bTileFound and fDistance < fMaxDistance:
+        if vRayLength1D[0] < vRayLength1D[1]:
+            vMapCheck[0] += vStep[0]
+            fDistance = vRayLength1D[0]
+            vRayLength1D[0] += vRayUnitStepSize[0]
+        elif vRayLength1D[0] > vRayLength1D[1]:
+            vMapCheck[1] += vStep[1]
+            fDistance = vRayLength1D[1]
+            vRayLength1D[1] += vRayUnitStepSize[1]
+        elif vRayLength1D[0] == vRayLength1D[1]:
+            vMapCheck += vStep
+            vRayLength1D += vRayUnitStepSize
+
+        if 0 <= vMapCheck[0] < w and 0 <= vMapCheck[1] < h:
+            if binary_map[int(vMapCheck[1]), int(vMapCheck[0])] == 1:
+                bTileFound = True
+        else:
+            break
+
+    return angle, fDistance, bTileFound
+
+
+@jit(nopython=True)
+def foo(x, y, angle, depth, w, h, binary_map, STEP_ANGLE, CASTED_RAYS):
+    return [
+        ray_cast_dda(x, y, angle + i * STEP_ANGLE, depth, w, h, binary_map)
+        for i in range(CASTED_RAYS)
+    ]
+
+
+if __name__ == "__main__":
+    start = time.time()
+    result = main(args)
+    end = time.time()
+    logging.info(f"DFP takes {end-start} s to output map.")
+
+    h, w = result.shape
+    bg = rgb_im = np.zeros((h, w, 3))
+    bg[result != 10] = [200, 200, 200]
+    bg[result == 10] = [100, 100, 100]
+    result[result != 10] = 0
+    result[result == 10] = 1
+    bg = np.transpose(bg, (1, 0, 2))
+    SCREEN_HEIGHT = h
+    SCREEN_WIDTH = w * 2
+    TILE_SIZE = 1
+    MAX_DEPTH = max(h, w)
+    SCALE = (SCREEN_WIDTH / 2) / args.casted_rays
+
+    # pdb.set_trace()
+    posy, posx = np.where(result != 1)
+    posidx = random.randint(0, len(posy) - 1)
+    player_x, player_y = posx[posidx], posy[posidx]
+    player_angle = math.pi
+    start_angle = math.pi
+    # deploy_plot_res(result)
+    # plt.show()
+    pygame.init()
+    win = pygame.display.set_mode((SCREEN_WIDTH, SCREEN_HEIGHT))
+    pygame.display.set_caption("Deep Floorplan Raycasting")
+
+    clock = pygame.time.Clock()
+    while True:
+        for event in pygame.event.get():
+            if event.type == pygame.QUIT:
+                pygame.quit()
+                sys.exit(0)
+
+        # 3d
+        pygame.draw.rect(win, (100, 100, 100), (w, h / 2, w, h))
+        pygame.draw.rect(win, (0, 150, 200), (w, -h / 2, w, h))
+
+        keys = pygame.key.get_pressed()
+        # control
+        player_angle, player_x, player_y = player_control(
+            keys, player_angle, player_x, player_y, w, h, result
+        )
+
+        # draw_map(result, win, TILE_SIZE, TILE_SIZE)
+        # draw floorplan
+        draw_dfp(bg, win)
+
+        # draw fps
+        show_fps(win, clock)
+
+        # draw player
+        pygame.draw.circle(win, (255, 0, 0), (player_x, player_y), 8)
+
+        # # draw player direction
+        wallangles = foo(
+            player_x,
+            player_y,
+            player_angle - HALF_FOV,
+            MAX_DEPTH,
+            w,
+            h,
+            result,
+            STEP_ANGLE,
+            args.casted_rays,
+        )
+
+        for idx, (angle, fdist, iswall) in enumerate(wallangles):
+            draw_player_direction(player_x, player_y, angle, fdist, win)
+            wall_color = int(255 * (1 - (3 * fdist / MAX_DEPTH) ** 2))
+            wall_color = max(min(wall_color, 255), 30)
+            fdist *= math.cos(player_angle - angle)
+            wall_height = 21000 / (fdist + 0.00001)
+
+            if iswall:
+                pygame.draw.rect(
+                    win,
+                    (wall_color, wall_color, wall_color),
+                    (
+                        int(w + idx * SCALE),
+                        int((h / 2) - wall_height / 2),
+                        SCALE,
+                        wall_height,
+                    ),
+                )
+        pygame.display.flip()
+
+        clock.tick()
+
+    pygame.quit()

src/dfp/game/controller.py

@@ -0,0 +1,11 @@
+import argparse
+
+from .model import Model
+from .view import View
+
+
+class Controller:
+    def __init__(self, config: argparse.Namespace):
+        self.config = config
+        self.model = Model(config)
+        self.view = View(config)

src/dfp/game/model.py

@@ -0,0 +1,6 @@
+import argparse
+
+
+class Model:
+    def __init__(self, config: argparse.Namespace):
+        self.config = config

src/dfp/game/view.py

@@ -0,0 +1,6 @@
+import argparse
+
+
+class View:
+    def __init__(self, config: argparse.Namespace):
+        self.config = config