Update Docker Compose files and fix code issues

dev/scripts/reinit_db.sh

@@ -1,10 +1,10 @@
 #!/usr/bin/env bash
 
 echo "Removing AppDB/saved_data..."
-rm -rf ../../services/AppDB/saved_data;
+rm -rf ../../services/base/AppDB/saved_data;
 echo "Removing ExpeDB/saved_data..."
-rm -rf ../../services/ExpeDB/saved_data;
+rm -rf ../../services/base/ExpeDB/saved_data;
 echo "Removing JupyterLab/Notebooks/Experiments..."
-find ../../services/JupyterLab/Notebooks/Experiments -type d ! -name '.*' -mindepth 1 -exec rm -rf {} +;
+find ../../services/base/JupyterLab/Notebooks/Experiments  -mindepth 1 -type d ! -name '.*'  -exec rm -rf {} +;
 echo "Done!"
 

services/base/AutoDiscServer/Dockerfile

@@ -41,12 +41,12 @@ RUN ln -s /etc/nginx/sites-available/flask /etc/nginx/sites-enabled/flask
 
 # AutoDiscServer dependency resolution
 # TODO: convert to poetry package
-COPY flask_app/app.py /usr/src/services/flask_app/app.py
+#COPY flask_app/app.py /usr/src/services/flask_app/app.py
 COPY flask_app/requirements.txt /usr/src/services/flask_app/requirements.txt
 RUN pip install -r /usr/src/services/flask_app/requirements.txt
 
 # stage app
-COPY flask_app/ /usr/src/services/flask_app/
+#COPY flask_app/ /usr/src/services/flask_app/
 # stage user custom modules in adtool_custom
 COPY libs/adtool_custom /usr/src/libs/adtool_custom
 WORKDIR /usr/src/libs
@@ -55,4 +55,4 @@ RUN poetry lock; poetry install
 
 # start daemons
 WORKDIR /usr/src/services/flask_app
-CMD gunicorn -w 4 -b 0.0.0.0:5201 app:app --access-logfile '-' --log-level 'debug' & nginx -g "daemon off;"
+CMD gunicorn -w 4 -b 0.0.0.0:5201 app:app --access-logfile '-' --log-level 'debug' --reload & nginx -g "daemon off;"

services/base/AutoDiscServer/flask_app/app.py

@@ -46,6 +46,7 @@ def list_experiments():
     return make_response(jsonify({"ID": experiment_ids}), 200)
 
 
+
 @app.route("/experiments", methods=["POST"])
 def create_experiment():
     try:
@@ -70,7 +71,6 @@ def stop_experiment(id):
 
 # Explorers
 
-
 @app.route("/explorers", methods=["GET"])
 def list_explorers():
     info = get_auto_disc_registered_modules_info(get_modules("explorers"))
@@ -144,4 +144,5 @@ def list_hosts():
 
 
 if __name__ == "__main__":
-    app.run(host=config.FLASK_HOST, port=config.FLASK_PORT)
+    app.run(host=config.FLASK_HOST, port=config.FLASK_PORT,
+            debug= True, use_reloader=True)

services/base/AutoDiscServer/flask_app/experiments/experiments_handler.py

@@ -7,7 +7,7 @@
 from utils import AutoDiscServerConfig, CheckpointsStatusEnum, reconstruct_parameters
 from utils.DB import AppDBCaller, AppDBMethods
 from utils.experiment_status_enum import ExperimentStatusEnum
-
+import sys
 
 class ExperimentsHandler:
     def __init__(self):
@@ -27,6 +27,7 @@ def _get_experiment(self, id):
             iter(filter(lambda element: element.id == id, self._experiments)), None
         )
         if experiment is None:
+            print("experiments", self._experiments, file=sys.stderr)
             raise Exception("Unknown experiment ID !")
 
         return experiment
@@ -56,6 +57,7 @@ def _create_experiment(self, id, parameters):
             )
 
         # Add it in the list
+
         self._experiments.append(experiment)
 
         return experiment
@@ -114,7 +116,6 @@ def add_experiment(self, parameters):
         try:
             # Add experiment in DB and obtain the id
             exp_date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M %Z")
-            print("Before response")
             request = {
                 "name": parameters["experiment"]["name"],
                 "created_on": exp_date,

services/base/AutoDiscServer/libs/adtool_default/systems/ExponentialMixture.py

@@ -90,9 +90,6 @@ def test():
     assert len(ExponentialMixture.CONFIG_DEFINITION) > 0
 
 
-print("ExponentialMixture", ExponentialMixture, file=sys.stderr)
 
 if __name__ == "__main__":
     test()
-
-print("ExponentialMixture.CONFIG_DEFINITION", ExponentialMixture.CONFIG_DEFINITION, file=sys.stderr)

services/base/AutoDiscServer/libs/auto_disc/ExperimentPipeline.py

@@ -199,8 +199,10 @@ def run(self, n_exploration_runs: int):
                     run_idx_start_from_one % self.save_frequency == 0
                     or run_idx_start_from_one == n_exploration_runs
                 ):
+                    print("############ saving", self.resource_uri)
                     self.save(resource_uri=self.resource_uri)
 
+
                 self.run_idx += 1
 
         except Exception:

services/base/AutoDiscServer/libs/auto_disc/auto_disc/utils/expose_config/defaults.py

@@ -84,9 +84,7 @@ def _dataclass_to_config_dict(cls) -> Dict:
 
         # inner function to recurse through the dataclass
         def recurse(dc: type, parent: str):
-            print("dc", dc, file=sys.stderr)
             for k, v in dc.__dataclass_fields__.items():
-                print("kv", k, v, file=sys.stderr)
                 # unwrap from the Field object
                 unwrap_v = v.default
 

services/base/AutoDiscServer/libs/auto_disc/utils/leafutils/leafintegrations/expedb_locators.py

@@ -16,14 +16,14 @@ def _initialize_checkpoint(entrypoint: str, dict: Dict = {}) -> str:
     """
     Inserts a Mongo document and retrieves the unique key generated
     """
-    response = json.loads(requests.post(entrypoint, json=dict).content)
+    response = requests.post(entrypoint, json=dict).json()
     return response["ID"]
 
 
 def _query_uid(resource_uri: str, uid: LeafUID) -> List[Dict]:
     uid_filter = {"uid": uid}
     uid_filter = _format_filter_from_dict(uid_filter)
-    response = json.loads(requests.get(resource_uri + "?filter=" + uid_filter).content)
+    response = requests.get(resource_uri + "?filter=" + uid_filter).json()
     return response
 
 
@@ -102,6 +102,7 @@ def retrieve(self, uid: "LeafUID", *args, **kwargs) -> bytes:
 
     def _retrieve_mongo_id(self, uid: LeafUID) -> str:
         response = _query_uid(self.resource_uri, uid)
+        print('_retrieve_mongo_id response', response)
         if len(response) == 0:
             mongo_id = _initialize_checkpoint(self.resource_uri)
         elif len(response) == 1:

services/base/AutoDiscServer/libs/auto_disc/utils/leafutils/leafintegrations/tests/test_expedb_locators.py

@@ -59,14 +59,21 @@ def mock_checkpoint():
 
 
 def test_ExpeDBLocator_store():
-    bin = b"123"
+    bin = b"123 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
     loc = ExpeDBLocator(resource_uri=RESOURCE_URI)
+    print('loc', loc)
     uid = loc.store(bin)
+    print('uid', uid)
 
     # test by making a manual API call
     mongo_id = loc._retrieve_mongo_id(uid)
+    print('mongo_id', mongo_id)
     response_bin = requests.get(RESOURCE_URI + "/" + mongo_id + "/metadata").content
+
     response_bin = codecs.decode(response_bin, encoding="base64")
+
+    print('response_bin', response_bin)
+
     assert response_bin == bin
 
     # TODO: test that duplicate inserts result in update
@@ -79,6 +86,8 @@ def test_ExpeDBLocator_retrieve():
 
     retrieved_bin = loc.retrieve(uid)
 
+    print("retrieved_bin", retrieved_bin)
+
     assert bin == retrieved_bin
 
 

services/base/AutoDiscServer/libs/debug_libs/debug.ipynb

@@ -0,0 +1,90 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "\n",
+    "sys.path.append('..')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import adtool_default"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Traceback (most recent call last):\n",
+      "  File \"<frozen runpy>\", line 198, in _run_module_as_main\n",
+      "  File \"<frozen runpy>\", line 88, in _run_code\n",
+      "  File \"/home/flowers-user/anaconda3/envs/adtool/lib/python3.11/unittest/__main__.py\", line 18, in <module>\n",
+      "    main(module=None)\n",
+      "  File \"/home/flowers-user/anaconda3/envs/adtool/lib/python3.11/unittest/main.py\", line 101, in __init__\n",
+      "    self.parseArgs(argv)\n",
+      "  File \"/home/flowers-user/anaconda3/envs/adtool/lib/python3.11/unittest/main.py\", line 150, in parseArgs\n",
+      "    self.createTests()\n",
+      "  File \"/home/flowers-user/anaconda3/envs/adtool/lib/python3.11/unittest/main.py\", line 161, in createTests\n",
+      "    self.test = self.testLoader.loadTestsFromNames(self.testNames,\n",
+      "                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "  File \"/home/flowers-user/anaconda3/envs/adtool/lib/python3.11/unittest/loader.py\", line 232, in loadTestsFromNames\n",
+      "    suites = [self.loadTestsFromName(name, module) for name in names]\n",
+      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "  File \"/home/flowers-user/anaconda3/envs/adtool/lib/python3.11/unittest/loader.py\", line 232, in <listcomp>\n",
+      "    suites = [self.loadTestsFromName(name, module) for name in names]\n",
+      "              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "  File \"/home/flowers-user/anaconda3/envs/adtool/lib/python3.11/unittest/loader.py\", line 162, in loadTestsFromName\n",
+      "    module = __import__(module_name)\n",
+      "             ^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "ValueError: Empty module name\n"
+     ]
+    }
+   ],
+   "source": [
+    "!python3 -m unittest ."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "adtool",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}