ci: Update cicd for cloudsql (#417)

.github/sync-repo-settings.yaml

@@ -34,7 +34,7 @@ branchProtectionRules:
       - "retrieval-service-app-pr (retrieval-app-testing)"
       - "retrieval-service-postgres-pr (retrieval-app-testing)"
       - "retrieval-service-alloydb-pr (retrieval-app-testing)"
-      - "retrieval-service-cloudsql-pr (retrieval-app-testing)"
+      - "retrieval-service-cloudsql-pg-pr (retrieval-app-testing)"
       - "llm-demo-langchain-tools-pr (retrieval-app-testing)"
       - "llm-demo-vertexai-fc-pr (retrieval-app-testing)"
 # Set team access

docs/datastore/cloudsql_postgres.md

@@ -51,7 +51,7 @@
     ```bash
     export DB_PASS=my-cloudsql-pass
     export DB_USER=postgres
-    export INSTANCE=my-cloudsql-instance
+    export INSTANCE=my-cloudsql-pg-instance
     export REGION=us-central1
     ```
 
@@ -101,7 +101,7 @@ datastore:
     # Update this with your project ID
     project: <PROJECT_ID>
     region: us-central1
-    instance: my-cloudsql-instance
+    instance: my-cloudsql-pg-instance
     # Update this with the database name
     database: "assistantdemo"
     # Update with database user, the default is `postgres`
@@ -158,7 +158,7 @@ Clean up after completing the demo.
 1. Delete the Cloud SQL instance:
 
     ```bash
-    gcloud sql instances delete my-cloudsql-instance
+    gcloud sql instances delete my-cloudsql-pg-instance
     ```
 
 ## Developer information
@@ -186,4 +186,4 @@ gcloud builds submit --config retrieval_service/cloudsql.tests.cloudbuild.yaml \
     --substitutions _DATABASE_NAME=$DB_NAME,_DATABASE_USER=$DB_USER,_CLOUDSQL_REGION=$DB_REGION,_CLOUDSQL_INSTANCE=$DB_INSTANCE
 ```
 
-Where `$DB_NAME`,`$DB_USER`,`$DB_REGION`,`$DB_CLUSTER`,`$DB_INSTANCE` are environment variables with your database values.
+Where `$DB_NAME`,`$DB_USER`,`$DB_REGION`,`$DB_CLUSTER`,`$DB_INSTANCE` are environment variables with your database values.

retrieval_service/cloudsql-mysql.tests.cloudbuild.yaml

@@ -0,0 +1,69 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+steps:
+  - id: Install dependencies
+    name: python:3.11
+    dir: retrieval_service
+    script: pip install -r requirements.txt -r requirements-test.txt --user
+
+  - id: Update config mysql
+    name: python:3.11
+    dir: retrieval_service
+    secretEnv:
+      - DB_USER
+      - DB_PASS
+    script: |
+        #!/usr/bin/env bash
+        # Create config
+        cp example-config-cloudsql.yml config.yml
+        sed -i "s/cloudsql-engine/cloudsql-mysql/g" config.yml
+        sed -i "s/my_database/${_DATABASE_NAME}/g" config.yml
+        sed -i "s/my-user/$$DB_USER/g" config.yml
+        sed -i "s/my-password/$$DB_PASS/g" config.yml
+        sed -i "s/my-project/$PROJECT_ID/g" config.yml
+        sed -i "s/my-region/${_CLOUDSQL_REGION}/g" config.yml
+        sed -i "s/my-instance/${_CLOUDSQL_INSTANCE}/g" config.yml
+
+  - id: Run Cloud SQL mysql DB integration tests
+    name: python:3.11
+    dir: retrieval_service
+    env: # Set env var expected by tests
+      - "DB_NAME=${_DATABASE_NAME}"
+      - "DB_PROJECT=$PROJECT_ID"
+      - "DB_REGION=${_CLOUDSQL_REGION}"
+      - "DB_INSTANCE=${_CLOUDSQL_INSTANCE}"
+    secretEnv:
+      - DB_USER
+      - DB_PASS
+    script: |
+        #!/usr/bin/env bash
+        python -m pytest --cov=datastore.providers.cloudsql_mysql --cov-config=coverage/.cloudsql-mysql-coveragerc datastore/providers/cloudsql_mysql_test.py
+
+substitutions:
+  _DATABASE_NAME: test_${SHORT_SHA}
+  _CLOUDSQL_REGION: "us-central1"
+  _CLOUDSQL_INSTANCE: "my-cloudsql-mysql-instance"
+
+availableSecrets:
+  secretManager:
+    - versionName: projects/$PROJECT_ID/secrets/cloudsql_mysql_pass/versions/latest
+      env: DB_PASS
+    - versionName: projects/$PROJECT_ID/secrets/cloudsql_mysql_user/versions/latest
+      env: DB_USER
+
+options:
+  automapSubstitutions: true
+  substitutionOption: 'ALLOW_LOOSE'
+  dynamic_substitutions: true

retrieval_service/cloudsql.tests.cloudbuild.yaml → retrieval_service/cloudsql-pg.tests.cloudbuild.yaml

@@ -18,7 +18,7 @@ steps:
     dir: retrieval_service
     script: pip install -r requirements.txt -r requirements-test.txt --user
 
-  - id: Update config
+  - id: Update config postgres
     name: python:3.11
     dir: retrieval_service
     secretEnv:
@@ -28,14 +28,15 @@ steps:
         #!/usr/bin/env bash
         # Create config
         cp example-config-cloudsql.yml config.yml
+        sed -i "s/cloudsql-engine/cloudsql-postgres/g" config.yml
         sed -i "s/my_database/${_DATABASE_NAME}/g" config.yml
         sed -i "s/my-user/$$DB_USER/g" config.yml
         sed -i "s/my-password/$$DB_PASS/g" config.yml
         sed -i "s/my-project/$PROJECT_ID/g" config.yml
         sed -i "s/my-region/${_CLOUDSQL_REGION}/g" config.yml
         sed -i "s/my-instance/${_CLOUDSQL_INSTANCE}/g" config.yml
 
-  - id: Run Cloud SQL DB integration tests
+  - id: Run Cloud SQL postgres DB integration tests
     name: python:3.11
     dir: retrieval_service
     env: # Set env var expected by tests
@@ -48,19 +49,18 @@ steps:
       - DB_PASS
     script: |
         #!/usr/bin/env bash
-        python -m pytest --cov=datastore.providers.cloudsql_postgres --cov-config=coverage/.cloudsql-coveragerc datastore/providers/cloudsql_postgres_test.py
+        python -m pytest --cov=datastore.providers.cloudsql_postgres --cov-config=coverage/.cloudsql-pg-coveragerc datastore/providers/cloudsql_postgres_test.py
 
 substitutions:
   _DATABASE_NAME: test_${SHORT_SHA}
-  _DATABASE_USER: postgres
   _CLOUDSQL_REGION: "us-central1"
-  _CLOUDSQL_INSTANCE: "my-cloudsql-instance"
+  _CLOUDSQL_INSTANCE: "my-cloudsql-pg-instance"
 
 availableSecrets:
   secretManager:
-    - versionName: projects/$PROJECT_ID/secrets/cloudsql_pass/versions/latest
+    - versionName: projects/$PROJECT_ID/secrets/cloudsql_pg_pass/versions/latest
       env: DB_PASS
-    - versionName: projects/$PROJECT_ID/secrets/cloudsql_user/versions/latest
+    - versionName: projects/$PROJECT_ID/secrets/cloudsql_pg_user/versions/latest
       env: DB_USER
 
 options:

retrieval_service/coverage/.cloudsql-pg-coveragerc

@@ -0,0 +1,9 @@
+[run]
+branch = true
+omit =
+    */__init__.py
+
+[report]
+show_missing = true
+precision = 2
+fail_under = 90

retrieval_service/datastore/providers/alloydb.py

@@ -27,7 +27,7 @@
 
 from .. import datastore
 
-POSTGRES_IDENTIFIER = "alloydb-postgres"
+ALLOYDB_PG_IDENTIFIER = "alloydb-postgres"
 
 
 class Config(BaseModel, datastore.AbstractConfig):
@@ -46,7 +46,7 @@ class Client(datastore.Client[Config]):
 
     @datastore.classproperty
     def kind(cls):
-        return "alloydb-postgres"
+        return ALLOYDB_PG_IDENTIFIER
 
     def __init__(self, pool: AsyncEngine):
         self.__pool = pool

retrieval_service/datastore/providers/cloudsql_postgres.py

@@ -27,7 +27,7 @@
 
 from .. import datastore
 
-POSTGRES_IDENTIFIER = "cloudsql-postgres"
+CLOUD_SQL_PG_IDENTIFIER = "cloudsql-postgres"
 
 
 class Config(BaseModel, datastore.AbstractConfig):
@@ -45,7 +45,7 @@ class Client(datastore.Client[Config]):
 
     @datastore.classproperty
     def kind(cls):
-        return "cloudsql-postgres"
+        return CLOUD_SQL_PG_IDENTIFIER
 
     def __init__(self, pool: AsyncEngine):
         self.__pool = pool

retrieval_service/datastore/providers/firestore.py

@@ -25,6 +25,8 @@
 
 from .. import datastore
 
+FIRESTORE_IDENTIFIER = "firestore"
+
 
 class Config(BaseModel, datastore.AbstractConfig):
     kind: Literal["firestore"]
@@ -36,7 +38,7 @@ class Client(datastore.Client[Config]):
 
     @datastore.classproperty
     def kind(cls):
-        return "firestore"
+        return FIRESTORE_IDENTIFIER
 
     def __init__(self, client: AsyncClient):
         self.__client = client

retrieval_service/datastore/providers/postgres.py

@@ -42,7 +42,7 @@ class Client(datastore.Client[Config]):
 
     @datastore.classproperty
     def kind(cls):
-        return "postgres"
+        return POSTGRES_IDENTIFIER
 
     def __init__(self, pool: asyncpg.Pool):
         self.__pool = pool

retrieval_service/example-config-cloudsql.yml

@@ -1,7 +1,7 @@
 host: 0.0.0.0
 datastore:
   # Example for Cloud SQL
-  kind: "cloudsql-postgres"
+  kind: "cloudsql-engine"
   project: "my-project"
   region: "my-region"
   instance: "my-instance"