devcontainers setup

.devcontainer/devcontainer.json

@@ -1,34 +1,68 @@
 {
-    "name": "depictio devcontainer",
-    "dockerComposeFile": ["../docker-compose.yaml"], // Adjusting the path
-    "service": "depictio_frontend",
-    "workspaceFolder": "/workspace",
-    "extensions": [
-        "ms-python.python",
-        "ms-azuretools.vscode-docker",
-        "ms-python.anaconda-extension-pack",
-        "GitHub.copilot"
-    ],
-    "settings": {
-        "terminal.integrated.shell.linux": "/bin/bash"
+    "name": "Depictio devcontainer",
+    // "dockerComposeFile": [
+    //     "../docker-compose.yml"
+    // ],
+    "image": "mcr.microsoft.com/devcontainers/python:1-3.11",
+    // "service": "depictio_frontend",
+    "workspaceFolder": "/workspaces/depictio",
+    "customizations": {
+        "vscode": {
+            "settings": {
+                "editor.tabSize": 2,
+                "git.detectSubmodules": false,
+                "terminal.integrated.defaultProfile.linux": "zsh",
+                "workbench.editor.wrapTabs": true
+            },
+            "extensions": [
+                "ms-python.python",
+                "ms-azuretools.vscode-docker",
+                "ms-python.anaconda-extension-pack",
+                "GitHub.copilot",
+                "charliermarsh.ruff",
+                "mongodb.mongodb-vscode"
+            ]
+        }
     },
-    "postCreateCommand": "conda env create -f conda_envs/depictio_cli.yaml && ./download_depictio_data.sh",
-    "forwardPorts": [
-        27018,
-        3000,
-        9010,
-        5080,
-        8058,
-        9000,
-        9001
-    ],
-    "remoteUser": "vscode",
+    "postCreateCommand": "./setup_codespaces.sh",
+    // "forwardPorts": [
+    //     27018,
+    //     5080,
+    //     8058,
+    //     9000,
+    //     9001
+    // ],
+    // "portsAttributes": {
+    //     "27018": {
+    //         "label": "mongo"
+    //     },
+    //     "5080": {
+    //         "label": "depictio_frontend"
+    //     },
+    //     "8058": {
+    //         "label": "depictio_backend"
+    //     },
+    //     "9000-9001": {
+    //         "label": "minio"
+    //     }
+    // },
     "remoteEnv": {
         "MINIO_ROOT_USER": "${localEnv:MINIO_ROOT_USER}",
         "MINIO_ROOT_PASSWORD": "${localEnv:MINIO_ROOT_PASSWORD}",
         "MINIO_ACCESS_KEY": "${localEnv:MINIO_ACCESS_KEY}",
         "MINIO_SECRET_KEY": "${localEnv:MINIO_SECRET_KEY}",
         "DEPICTIO_BACKEND_DATA_VOLUME_HOST": "/workspace/depictio-data",
-        "AUTH_TMP_TOKEN": "${localEnv:AUTH_TMP_TOKEN}"
+        "AUTH_PRIVATE_KEY": "${localEnv:AUTH_PRIVATE_KEY}",
+        "AUTH_PUBLIC_KEY": "${localEnv:AUTH_PUBLIC_KEY}"
+    },
+    "features": {
+        "ghcr.io/devcontainers/features/common-utils:2": {
+            "installZsh": "true",
+            "installGit": "true",
+            "installDocker": "true"
+        },
+        "ghcr.io/devcontainers/features/docker-in-docker:2": {
+            "version": "latest"
+        }
     }
 }

.gitignore

@@ -194,7 +194,7 @@ depictio/private_key.pem
 depictio/public_key.pem
 jbrowse2/
 delta_dev/
-dev/dev_minio/
+!dev/dev_minio/dev.ipynb
 dev/dev_design_visu/data/
 depictio/jbrowse2_configs/
 jbrowse2_sessions/
@@ -214,3 +214,7 @@ dev/dev_delta/.DS_Store
 .DS_Store
 dev/dev_delta/.DS_Store
 jbrowse-watcher-plugin
+
+# Depictio-CLI venv
+depictio-cli-venv/
+dev/dev_minio/my_table/

configs/mosaicatcher_pipeline/mosaicatcher_pipeline.yaml

@@ -27,8 +27,8 @@ workflows:
       # TODO: to turn into a dict (location - regex)
       # The location of the runs - multiple locations can be specified
       parent_runs_location:
-        - "/Users/tweber/Data/mosaicatcher-pipeline"
-        # - "/Users/tweber/Gits/depictio-data/StrandSeq/HGSVC"
+        # - "/Users/tweber/Data/mosaicatcher-pipeline"
+        - "./depictio-data/StrandSeq/HGSVC"
 
       # Regular expression to match the runs
       runs_regex: ".*"
@@ -107,65 +107,65 @@ workflows:
             polars_kwargs:
               separator: "\t"
 
-      - data_collection_tag: "mosaicatcher_samples_metadata"
-        description: "Metadata file for MosaiCatcher samples"
-        config:
-          type: "Table"
-          metatype: "Metadata"
-          regex: 
-            pattern: "mosaicatcher_samples_metadata.csv"
-            type: "file-based"
-          dc_specific_properties:
-            format: "CSV"
-            polars_kwargs:
-              separator: ";"
-          join:
-            # The columns that are used to join the data collections
-            on_columns:
-              - "sample"
-            # The type of join that is used to join the data collections
-            how: "inner"
-            # The list of data collections that are used to join the data collection
-            with_dc:
-              - "ashleys_labels"
-              - "mosaicatcher_stats"
+      # - data_collection_tag: "mosaicatcher_samples_metadata"
+      #   description: "Metadata file for MosaiCatcher samples"
+      #   config:
+      #     type: "Table"
+      #     metatype: "Metadata"
+      #     regex: 
+      #       pattern: "mosaicatcher_samples_metadata.csv"
+      #       type: "file-based"
+      #     dc_specific_properties:
+      #       format: "CSV"
+      #       polars_kwargs:
+      #         separator: ";"
+      #     join:
+      #       # The columns that are used to join the data collections
+      #       on_columns:
+      #         - "sample"
+      #       # The type of join that is used to join the data collections
+      #       how: "inner"
+      #       # The list of data collections that are used to join the data collection
+      #       with_dc:
+      #         - "ashleys_labels"
+      #         - "mosaicatcher_stats"
 
 
       # Data collections of type JBrowse2
-      - data_collection_tag: "SV_calls"
-
-        # A description of the data collection
-        description: "BED files containing SV calls generated by MosaiCatcher"
-
-        # The configuration of the data collection
-        config:
-          # The type of the data collection
-          type: "JBrowse2"
-
-          # File format of the files in the data collection
-          format: "BED"
-
-          # A regular expression that is used to match the files specific of the data collection
-          regex: 
-            pattern: "{cell}-SV.bed.gz"
-            wildcards: 
-              - name: "cell"
-                wildcard_regex: ".*"
-              # - value: ...
-            type: "file-based"
-
-          join:
-            with_dc: 
-              - "mosaicatcher_stats"
-            how: "inner"
-            on_columns:
-              - "cell"
-
-          # The specific properties of the data collection - is related to the type of the data collection (JBrowse2 here)
-          dc_specific_properties:
-
-            # The extension of the index file for the data collection
-            index_extension: "tbi"
-
-            # The location of the JBrowse template for the data collection
-            jbrowse_template_location: "/configs/mosaicatcher_pipeline/jbrowse2_templates/SV_calls.json"
+      # - data_collection_tag: "SV_calls"
+
+      #   # A description of the data collection
+      #   description: "BED files containing SV calls generated by MosaiCatcher"
+
+      #   # The configuration of the data collection
+      #   config:
+      #     # The type of the data collection
+      #     type: "JBrowse2"
+
+      #     # File format of the files in the data collection
+      #     format: "BED"
+
+      #     # A regular expression that is used to match the files specific of the data collection
+      #     regex: 
+      #       pattern: "{cell}-SV.bed.gz"
+      #       wildcards: 
+      #         - name: "cell"
+      #           wildcard_regex: ".*"
+      #         # - value: ...
+      #       type: "file-based"
+
+      #     join:
+      #       with_dc: 
+      #         - "mosaicatcher_stats"
+      #       how: "inner"
+      #       on_columns:
+      #         - "cell"
+
+      #     # The specific properties of the data collection - is related to the type of the data collection (JBrowse2 here)
+      #     dc_specific_properties:
+
+      #       # The extension of the index file for the data collection
+      #       index_extension: "tbi"
+
+      #       # The location of the JBrowse template for the data collection
+      #       jbrowse_template_location: "/configs/mosaicatcher_pipeline/jbrowse2_templates/SV_calls.json"

depictio-cli/depictio_cli/depictio_cli.py

@@ -369,14 +369,21 @@ def setup(
         # check if token exists in the config file
         config = load_depictio_config()
         token = config.get("token")
+        print(config)
+        print(token)
         if not token:
             typer.echo("A valid token must be provided for authentication.")
             raise typer.Exit(code=1)
 
+    # Delete bucket 
+    # response = httpx.get(f"{API_BASE_URL}/depictio/api/v1/utils/delete_bucket")
+    # print(response.json())
+
     # Create bucket if not exists
     response = httpx.get(f"{API_BASE_URL}/depictio/api/v1/utils/create_bucket")
     print(response.json())
 
+
     if erase_all:
         # Drop all collections
         response = httpx.get(f"{API_BASE_URL}/depictio/api/v1/utils/drop_all_collections")
@@ -391,6 +398,8 @@ def setup(
         raise typer.Exit(code=1)
 
     user = return_user_from_token(token)  # Decode the token to get the user information
+    print(user)
+
     if not user:
         typer.echo("Invalid token or unable to decode^ user information.")
         raise typer.Exit(code=1)

depictio/api/v1/configs/config.py

@@ -15,3 +15,15 @@
 TOKEN = settings.auth.tmp_token
 
 
+# Load your private key from the settings
+PRIVATE_KEY = settings.auth.private_key.encode() if settings.auth.private_key else None
+# Load your public key from the settings
+PUBLIC_KEY = settings.auth.public_key.encode() if settings.auth.public_key else None
+
+ALGORITHM = "RS256"
+ACCESS_TOKEN_EXPIRE_MINUTES = 360 * 3600
+
+if not PRIVATE_KEY or not PUBLIC_KEY:
+    logger.error("Private or public key not found in environment variables.")
+else:
+    logger.info("Private and public keys successfully loaded.")

depictio/api/v1/configs/create_user_mongodb.py

@@ -0,0 +1,21 @@
+import pymongo
+from depictio.api.v1.configs.config import settings
+
+# FIXME: This is a temporary solution to avoid the error: "pymongo.errors.ServerSelectionTimeoutError: localhost:27017: [Errno 111] Connection refused"
+MONGODB_URL = f"mongodb://localhost:{settings.mongodb.port}/"
+
+# Set up the MongoDB client
+client = pymongo.MongoClient(MONGODB_URL)
+db = client[settings.mongodb.db_name]
+
+# Access the collections
+users_collection = db[settings.mongodb.collections.users_collection]
+
+# Insert a new document into the users collection
+new_user = {
+    "username": "Cezanne",
+    "password": "Paul"
+}
+
+result = users_collection.insert_one(new_user)
+print(f"Inserted user with _id: {result.inserted_id}")

depictio/api/v1/configs/settings_models.py

@@ -88,6 +88,8 @@ class Config:
 class Auth(BaseSettings):
     """Authentication configuration."""
     tmp_token: str = Field(default="eyJhb...")
+    private_key: str = Field(default=None)
+    public_key: str = Field(default=None)
 
     class Config:
         env_prefix = 'AUTH_'

depictio/api/v1/deltatables_utils.py

@@ -1,9 +1,8 @@
 from bson import ObjectId
 import httpx
 import polars as pl
-import pandas as pd
 from depictio.api.v1.configs.config import API_BASE_URL, TOKEN
-from depictio.api.v1.s3 import s3_client, minio_storage_options
+from depictio.api.v1.s3 import minio_storage_options
 from depictio.api.v1.configs.config import logger
 
 

depictio/api/v1/endpoints/dashboards_endpoints/models.py

@@ -1,14 +1,7 @@
-from typing import Dict, List, Optional, Union
-import bleach
-import re
+from typing import Dict, List, Optional
 from bson import ObjectId
-from pydantic import (
-    BaseModel,
-    Field,
-    validator,
-)
 
-from depictio.api.v1.models.base import MongoModel, PyObjectId
+from depictio.api.v1.models.base import MongoModel
 
 class DashboardData(MongoModel):
     dashboard_id: str