Merge pull request #36 from SINTEF-9012/learning-factory

Learning factory

.devcontainer/Dockerfile

@@ -0,0 +1 @@
+FROM python:3.10-slim-bullseye

.devcontainer/devcontainer.json

@@ -0,0 +1,61 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.238.0/containers/python-3
+{
+	"name": "SINDIT Devcontainer (Fullstack, Python-based)",
+	// Docker-compose used to utilize the docker network etc. with the
+	// usual syntax
+	"dockerComposeFile": [
+		"docker-compose.yml"
+	],
+	// The 'service' property is the name of the service for the container that VS Code should
+	// use. Update this value and .devcontainer/docker-compose.yml to the real service name.
+	"service": "sindit_devcontainer",
+	// The optional 'workspaceFolder' property is the path VS Code should open by default when
+	// connected. This is typically a file mount in .devcontainer/docker-compose.yml
+	"workspaceFolder": "/workspace",
+	// Install requirements (here instead of the Dockerfile, to be able to use the workspace)
+	"postCreateCommand": "./container-requirements-initializer.sh && printf '\n\n\n\n\nTo activate code analysis, execute:\n\n\t > Developer: Reload Window\n\n\n\n\n'",
+	// Configure tool-specific properties.
+	"customizations": {
+		// Configure properties specific to VS Code.
+		"vscode": {
+			// Set *default* container specific settings.json values on container create.
+			"settings": {
+				"python.defaultInterpreterPath": "/usr/local/bin/python",
+				"python.linting.enabled": true,
+				"python.linting.pylintEnabled": true,
+				"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+				"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+				"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
+				"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
+				"python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
+				"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
+				"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
+				"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
+				"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint"
+			},
+			// Add the IDs of extensions you want installed when the container is created.
+			"extensions": [
+				"ms-python.python",
+				"ms-python.vscode-pylance",
+				"ms-azuretools.vscode-docker",
+				"njpwerner.autodocstring",
+				"visualstudioexptteam.vscodeintellicode",
+				"kevinrose.vsc-python-indent",
+				"donjayamanne.python-environment-manager",
+				"jakeboone02.cypher-query-language",
+				"tomoki1207.pdf",
+				"visualstudioexptteam.intellicode-api-usage-examples"
+			]
+		}
+	},
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "pip3 install --user -r requirements.txt",
+	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	// "remoteUser": "vscode",
+	"features": {
+		"docker-from-docker": "latest"
+	}
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,51 @@
+version: '2.4'
+
+networks:
+  # Same as the main docker composefor the devcontainer
+  sindit_devcontainer_network:
+    name: sindit_devcontainer_network
+
+services:
+  # Update this to the name of the service you want to work with in your docker-compose.yml file
+  sindit_devcontainer:
+    build: .
+    hostname: sindit-devcontainer
+    container_name: sindit_devcontainer
+    mem_limit: 8G
+    # GPU usage
+    # shm_size: '2gb'
+    # deploy:
+    #   resources:
+    #     reservations:
+    #       devices:
+    #         - capabilities: [ gpu ]
+    networks:
+      - sindit_devcontainer_network
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+    # If you want add a non-root user to your Dockerfile, you can use the "remoteUser"
+    # property in devcontainer.json to cause VS Code its sub-processes (terminals, tasks, 
+    # debugging) to execute as the user. Uncomment the next line if you want the entire 
+    # container to run as this user instead. Note that, on Linux, you may need to 
+    # ensure the UID and GID of the container user you create matches your local user. 
+    # See https://aka.ms/vscode-remote/containers/non-root for details.
+    #
+    # user: vscode
+
+
+    volumes:
+      # Mount the folder of your project
+      - ..:/workspace:cached
+
+      # Use Docker from inside the container. See https://aka.ms/vscode-remote/samples/docker-from-docker-compose for details.
+      - /var/run/docker.sock:/var/run/docker.sock
+
+    # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust.
+    # cap_add:
+    #   - SYS_PTRACE
+    # security_opt:
+    #   - seccomp:unconfined
+
+    # Overrides default command so things don't shut down after the process ends.
+    command: /bin/sh -c "while sleep 1000; do :; done"

.github/vpn/config.ovpn

@@ -0,0 +1,20 @@
+client
+dev tun
+proto tcp
+remote 78.91.103.68 443 tcp-client
+verify-x509-name "minicloud-openvpn-pfsense-servercertificate" name
+route remote_host 255.255.255.255 net_gateway
+resolv-retry infinite
+nobind
+persist-key
+persist-tun
+auth-user-pass secret.txt
+cipher AES-256-CBC
+auth SHA256
+comp-lzo no
+route-delay 4
+verb 3
+reneg-sec 0
+ca ca.crt
+cert user.crt
+key user.key

.github/workflows/docker-deployment-learning-factory.yaml

@@ -0,0 +1,64 @@
+name: Build and deploy docker-compose containers
+
+on:
+  push:
+    branches: [ maxcloud_deployment ]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    environment: maxcloud_deployment_environment
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install VPN
+      run: |
+        sudo apt-get update
+        sudo apt-get --assume-yes --no-install-recommends install openvpn
+  
+    - name: Setup VPN connection
+      run: |
+            echo "${{ secrets.MAXCLOUD_DEPLOYMENT_CA_CRT }}" > ca.crt
+            echo "${{ secrets.MAXCLOUD_DEPLOYMENT_USER_CRT }}" > user.crt
+            echo "${{ secrets.MAXCLOUD_DEPLOYMENT_USER_KEY }}" > user.key
+            echo "${{ secrets.MAXCLOUD_DEPLOYMENT_SECRET_USERNAME_PASSWORD }}" > secret.txt
+            echo "${{ secrets.MAXCLOUD_DEPLOYMENT_TLS_KEY }}" > tls.key  
+    
+    - name: Start VPN connection
+      run: |
+            sudo openvpn --config ".github/vpn/config.ovpn" --log "vpn.log" --daemon
+
+    - name: Wait for VPN connection
+      run: sleep 30
+
+    - name: Build containers
+      run: |
+        ssh sintef@192.168.1.6 "
+          # Pull the current release
+          cd ~/learning-factory-digital-twin/SINDIT;
+          git remote add origin https://github.com/SINTEF-9012/SINDIT.git;
+          git checkout learning-factory-deployment;
+          git config pull.rebase false;
+          git pull origin learning-factory-deployment;
+          git remote remove origin;
+
+          # Rebuild the local image(s)
+          docker-compose build
+
+          # (Re-)start the containers
+          docker-compose up -d
+        "
+    
+    - name: Stop VPN connection
+      if: always()
+      run: |
+        sudo chmod 777 vpn.log
+        sudo killall openvpn     
+
+    - name: Upload VPN logs
+      uses: actions/upload-artifact@v2
+      if: always()
+      with:
+        name: VPN logs
+        path: vpn.log

.gitignore

@@ -1,7 +1,14 @@
 *.pyc
-.vscode/*
 docker/
 .idea/
 *.aasx
 *.xml
 *.aas
+docker_mounted_storage/
+docker_mounted_storage_devcontainer/
+backups/
+dependencies/
+mm_test/yolov3_mobilenetv2_320_300e_coco_20210719_215349-d18dff72.pth
+mm_test/yolov3_mobilenetv2_320_300e_coco.py
+mmcv_models/faster_rcnn/faster_rcnn_x101_64x4d_fpn_1x_coco_20200204-833ee192.pth
+mmcv_models/faster_rcnn/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth

.vscode/launch.json

@@ -4,13 +4,59 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "DT Frontend",
+            "type": "python",
+            "request": "launch",
+            "program": "./dt_frontend.py",
+            "console": "integratedTerminal",
+            "justMyCode": true,
+            "envFile": "${workspaceFolder}/environment_and_configuration/devcontainer_environment_frontend.env"
+        },
+        {
+            "name": "DT Backend",
+            "type": "python",
+            "request": "launch",
+            "program": "./dt_backend.py",
+            "console": "integratedTerminal",
+            "justMyCode": true,
+            "envFile": "${workspaceFolder}/environment_and_configuration/devcontainer_environment_backend.env"
+        },
+        {
+            "name": "Similarity pipeline",
+            "type": "python",
+            "request": "launch",
+            "program": "./similarity_pipeline.py",
+            "console": "integratedTerminal",
+            "justMyCode": true,
+            "envFile": "${workspaceFolder}/environment_and_configuration/devcontainer_environment_backend.env"
+        },
+        {
+            "name": "Learning factory instance: initialization",
+            "type": "python",
+            "request": "launch",
+            "program": "./init_learning_factory_from_cypher_file.py",
+            "console": "integratedTerminal",
+            "justMyCode": true,
+            "envFile": "${workspaceFolder}/environment_and_configuration/devcontainer_environment_backend.env"
+        },
+        {
+            "name": "Learning factory: continuous ordering",
+            "type": "python",
+            "request": "launch",
+            "program": "./learning_factory_continuous_ordering.py",
+            "console": "integratedTerminal",
+            "justMyCode": true,
+            "envFile": "${workspaceFolder}/environment_and_configuration/devcontainer_environment_backend.env"
+        },
         {
             "name": "Python: Current File",
             "type": "python",
             "request": "launch",
             "program": "${file}",
             "console": "integratedTerminal",
-            "justMyCode": true
+            "justMyCode": true,
+            "envFile": "${workspaceFolder}/environment_and_configuration/devcontainer_environment_backend.env"
         }
     ]
 }

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "python.formatting.provider": "black",
+    "python.formatting.blackPath": "black",
+    "editor.formatOnSave": true
+}

Dockerfile

@@ -1,28 +1,12 @@
-FROM python:3.8-slim-buster
-
-# Set labels 
-LABEL vendor=SINTEF_Digital \
-      SINDIT.is-beta=True\
-      SINDIT.version="0.0.1-beta" \
-      SINDIT.release-date="2021-05-06"
+FROM python:3.10-slim-bullseye
 
 RUN mkdir /opt/sindit
 WORKDIR /opt/sindit
 ENV PYTHONPATH /opt/sindit
-	  
+
 COPY . .
 
-RUN pip install --no-cache-dir -r requirements.txt
-RUN apt-get update && apt-get install -y curl && apt-get clean
+RUN ./container-requirements-initializer.sh
 
 EXPOSE 8050
 EXPOSE 8000
-
-ENTRYPOINT ["/bin/bash", "/opt/sindit/main.sh" ]
-
-
-
-
-
-
-

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2021 SINTEF-9012
+Copyright (c) 2022 SINTEF-9012
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal