Release 0.4.60

.devcontainer/Dockerfile

@@ -0,0 +1,6 @@
+# Pull base image.
+FROM aperturedata/aperturedb-notebook:dependencies
+
+RUN python -m pip install --no-cache-dir \
+    awscli==1.45.11 \
+    openai-clip

.devcontainer/devcontainer.json

@@ -9,6 +9,7 @@
         "vscode": {
             "extensions": [
                 "ms-python.python",
+                "ms-python.vscode-pylance",
                 "ms-python.pylint",
                 "ms-toolsai.jupyter"
             ]

.devcontainer/docker-compose.yml

@@ -74,7 +74,8 @@ services:
       - ./aperturedb/certificate:/etc/nginx/certificate
   devcontainer:
     build:
-      context: ../docker/devcontainer
+      context: .
+      dockerfile: Dockerfile
     environment:
       DB_HOST: lenz
       DB_PORT: 55551

.github/workflows/develop.yml

@@ -28,12 +28,14 @@ jobs:
         username: ${{ secrets.DOCKER_USER }}
         password: ${{ secrets.DOCKER_PASS }}
 
-    - name: Login to Google Cloud
-      uses: google-github-actions/setup-gcloud@v0
+    - name: Authenticate to Google Cloud
+      uses: google-github-actions/auth@v2
       with:
-        service_account_key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
-        project_id:          ${{ secrets.GCP_SERVICE_ACCOUNT_PROJECT_ID }}
-        export_default_credentials: true
+        credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+        project_id: ${{ secrets.GCP_SERVICE_ACCOUNT_PROJECT_ID }}
+
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@v2
 
     - name: Build and Run Tests
       env:

.github/workflows/main.yml

@@ -26,12 +26,14 @@ jobs:
         username: ${{ secrets.DOCKER_USER }}
         password: ${{ secrets.DOCKER_PASS }}
 
-    - name: Login to Google Cloud
-      uses: google-github-actions/setup-gcloud@v0
+    - name: Authenticate to Google Cloud
+      uses: google-github-actions/auth@v2
       with:
-        service_account_key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
-        project_id:          ${{ secrets.GCP_SERVICE_ACCOUNT_PROJECT_ID }}
-        export_default_credentials: true
+        credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+        project_id: ${{ secrets.GCP_SERVICE_ACCOUNT_PROJECT_ID }}
+
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@v2
 
     - name: Build and Run Tests
       env:

.github/workflows/nightly.yml

@@ -0,0 +1,57 @@
+name: nightly
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-test:
+    runs-on:
+    - benchmark
+
+    steps:
+
+    - uses: actions/checkout@v3
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKER_USER }}
+        password: ${{ secrets.DOCKER_PASS }}
+
+    - name: Authenticate to Google Cloud
+      uses: google-github-actions/auth@v2
+      with:
+        credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+        project_id: ${{ secrets.GCP_SERVICE_ACCOUNT_PROJECT_ID }}
+
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@v2
+    - name: Build and Run Tests
+      env:
+        # Run both protocols inside a single job. run_test_container.sh starts
+        # the http and non_http stacks in parallel, so wall-clock time is
+        # roughly max(http, non_http) instead of the previous matrix approach
+        # which duplicated every docker image build (notebook deps / notebook
+        # / tests / coverage) on a second runner.
+        TEST_PROTOCOL: both
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+        RUNNER_NAME: ${{ runner.name }}_nightly
+        # BuildKit + inline cache so subsequent runs actually reuse layers
+        # pulled via --cache-from in ci.sh.
+        DOCKER_BUILDKIT: 1
+        COMPOSE_DOCKER_CLI_BUILD: 1
+        ADB_REPO: aperturedata/aperturedb
+        ADB_TAG: dev
+        LENZ_REPO: aperturedata/lenz
+        LENZ_TAG: dev
+      run: RUN_TESTS=true ./ci.sh
+      shell: bash
+

.github/workflows/pr.yaml

@@ -17,7 +17,7 @@ jobs:
     steps:
 
     - name: Cleanup previous run
-      run: docker run --rm -v ${{ github.workspace }}:/workspace alpine sh -c "rm -rf /workspace/test/aperturedb/db* /workspace/test/aperturedb/logs*"
+      run: docker run --rm -v ${{ github.workspace }}:/workspace alpine sh -c "rm -rf /workspace/test/aperturedb"
       continue-on-error: true
 
     - uses: actions/checkout@v3
@@ -28,12 +28,14 @@ jobs:
         username: ${{ secrets.DOCKER_USER }}
         password: ${{ secrets.DOCKER_PASS }}
 
-    - name: Login to Google Cloud
-      uses: google-github-actions/setup-gcloud@v0
+    - name: Authenticate to Google Cloud
+      uses: google-github-actions/auth@v2
       with:
-        service_account_key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
-        project_id:          ${{ secrets.GCP_SERVICE_ACCOUNT_PROJECT_ID }}
-        export_default_credentials: true
+        credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+        project_id: ${{ secrets.GCP_SERVICE_ACCOUNT_PROJECT_ID }}
+
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@v2
 
     - name: Build and Run Tests
       env:
@@ -65,5 +67,53 @@ jobs:
         ADB_TAG: dev
         LENZ_REPO: aperturedata/lenz
         LENZ_TAG: dev
+        SKIP_SLOW_TESTS: true
       run: ./ci.sh
       shell: bash
+
+  run_test_conda_gpu:
+
+    runs-on:
+    - self-hosted
+    - deployer
+
+    steps:
+
+    - name: Cleanup previous run
+      run: docker run --rm -v ${{ github.workspace }}:/workspace alpine sh -c "rm -rf /workspace/test/aperturedb"
+      continue-on-error: true
+
+    - uses: actions/checkout@v3
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKER_USER }}
+        password: ${{ secrets.DOCKER_PASS }}
+
+    - name: Authenticate to Google Cloud
+      uses: google-github-actions/auth@v2
+      with:
+        credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+        project_id: ${{ secrets.GCP_SERVICE_ACCOUNT_PROJECT_ID }}
+
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@v2
+    - name: Build tests on pytorch GPU image
+      run: |
+        rm -rf docker/pytorch-gpu/aperturedata
+        mkdir -p docker/pytorch-gpu/aperturedata
+        cp -r aperturedb pyproject.toml README.md test docker/pytorch-gpu/aperturedata
+        bash docker/pytorch-gpu/build.sh
+      shell: bash
+
+    - name: Run Tests
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+        RUNNER_NAME: ${{ runner.name }}_${{ github.run_id }}
+      run: |
+        cd test
+        ./run_test_container.sh aperturedata/aperturedb-pytorch-gpu

.github/workflows/release.yaml

@@ -30,12 +30,14 @@ jobs:
         username: ${{ secrets.DOCKER_USER }}
         password: ${{ secrets.DOCKER_PASS }}
 
-    - name: Login to Google Cloud
-      uses: google-github-actions/setup-gcloud@v0
+    - name: Authenticate to Google Cloud
+      uses: google-github-actions/auth@v2
       with:
-        service_account_key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
-        project_id:          ${{ secrets.GCP_SERVICE_ACCOUNT_PROJECT_ID }}
-        export_default_credentials: true
+        credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+        project_id: ${{ secrets.GCP_SERVICE_ACCOUNT_PROJECT_ID }}
+
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@v2
 
     - name: Build and Run Tests
       env:

aperturedb/CommonLibrary.py

@@ -271,7 +271,8 @@ def execute_query(client: Connector, query: Commands,
                   blobs: Blobs = [],
                   success_statuses: list[int] = [0],
                   response_handler: Optional[Callable] = None, commands_per_query: int = 1, blobs_per_query: int = 0,
-                  strict_response_validation: bool = False, cmd_index=None) -> Tuple[int, CommandResponses, Blobs]:
+                  strict_response_validation: bool = False, cmd_index=None,
+                  error_handler: Optional[Callable] = None) -> Tuple[int, CommandResponses, Blobs]:
     """
     Execute a batch of queries, doing useful logging around it.
     Calls the response handler if provided.
@@ -288,6 +289,8 @@ def execute_query(client: Connector, query: Commands,
         commands_per_query (int, optional): The number of commands per query. Defaults to 1.
         blobs_per_query (int, optional): The number of blobs per query. Defaults to 0.
         strict_response_validation (bool, optional): Whether to strictly validate the response. Defaults to False.
+        cmd_index (int, optional): The index of the command or batch, passed to the response handler. Defaults to None.
+        error_handler (Callable, optional): Callback invoked when the query returns an unexpected status or fails. Expected signature is `error_handler(query, response, blobs)`. Defaults to None.
 
     Returns:
         int: The result code.
@@ -310,14 +313,21 @@ def execute_query(client: Connector, query: Commands,
                 map_response_to_handler(response_handler,
                                         query, blobs, r, b, commands_per_query, blobs_per_query,
                                         cmd_index)
-            except BaseException as e:
+            except Exception as e:
                 logger.exception(e)
                 if strict_response_validation:
-                    raise e
+                    raise
     else:
         # Transaction failed entirely.
+        num_commands = len(query) if isinstance(query, list) else 1
+        truncated_query = query[:2] if isinstance(
+            query, list) and num_commands > 2 else query
+        query_summary = (
+            f"{num_commands} commands (showing first 2: {truncated_query})"
+            if num_commands > 2 else str(query)
+        )
         logger.error(
-            f"Failed query = {query} with response = {censor_tokens(r)}")
+            f"Failed query = {query_summary} with response = {censor_tokens(r)}")
         result = 1
 
     statuses = {}
@@ -334,16 +344,25 @@ def execute_query(client: Connector, query: Commands,
 
     # last_query_ok means result status >= 0
     if result != 1:
-        warn_list = []
+        warnings_count = 0
         for status, results in statuses.items():
             if status not in success_statuses:
-                for wr in results:
-                    warn_list.append(wr)
-        if len(warn_list) != 0:
+                warnings_count += len(results)
+        if warnings_count != 0:
             logger.warning(
-                f"Partial errors:\r\n{json.dumps(query, default=str)}\r\n{json.dumps(censor_tokens(warn_list), default=str)}")
+                f"Encountered {warnings_count} partial errors. "
+                "Use error_handler or inspect response for details."
+            )
             result = 2
 
+    if result != 0 and error_handler is not None:
+        try:
+            error_handler(query, r, blobs)
+        except Exception as e:
+            logger.exception(e)
+            if strict_response_validation:
+                raise
+
     return result, r, b
 
 
@@ -375,8 +394,7 @@ def map_response_to_handler(handler, query, query_blobs,  response, response_blo
         handler(
             query[start:end],
             query_blobs[blobs_start:blobs_end],
-            response[start:end] if isinstance(
-                response, list) else response,
+            response[start:end] if is_list else response,
             response_blobs[blobs_returned:blobs_returned + b_count] if
             len(response_blobs) >= blobs_returned + b_count else None,
             None if cmd_index_offset is None else cmd_index_offset + i)