Add mysql support

Signed-off-by: Burak Ince <burak.ince@linux.org.tr>
burakince · Aug 10, 2022 · aae35a8 · aae35a8
1 parent e705b22
commit aae35a8
Show file tree

Hide file tree

Showing 15 changed files with 484 additions and 52 deletions.
diff --git a/.env b/.env
@@ -1,3 +1,4 @@
 POSTGRES_VERSION=14.4
+MYSQL_VERSION=8.0.30
 MINIO_VERSION=RELEASE.2022-08-08T18-34-09Z
 FAKE_GCS_SERVER_VERSION=1.38.3
diff --git a/docker-compose.aws-mysql-test.yaml b/docker-compose.aws-mysql-test.yaml
@@ -0,0 +1,67 @@
+version: "3.8"
+
+services:
+  minio:
+    container_name: minio-mysql-test
+    hostname: minio
+    image: minio/minio:${MINIO_VERSION}
+    entrypoint: sh
+    command: -c 'mkdir -p /data/mlflow && minio server /data --console-address ":9001"'
+    environment:
+      MINIO_ROOT_USER: minioadmin
+      MINIO_ROOT_PASSWORD: minioadmin
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    volumes:
+      - minio-mysql-storage:/data
+    networks:
+      - aws_mysql_test_nw
+
+  mysql:
+    image: "mysql:${MYSQL_VERSION}"
+    container_name: mlflow-aws-mysql-db
+    environment:
+      MYSQL_ROOT_PASSWORD: password
+      MYSQL_DATABASE: mlflow
+      MYSQL_USER: mlflow
+      MYSQL_PASSWORD: password
+      MYSQL_TCP_PORT: 3306
+    ports:
+      - "3306:3306"
+    volumes:
+      - mysql-aws-storage:/var/lib/mysql
+    networks:
+      - aws_mysql_test_nw
+
+  mlflow:
+    container_name: mlflow-aws-mysql-test
+    build:
+      context: .
+      dockerfile: Dockerfile
+    command: "mlflow server --backend-store-uri=mysql+pymysql://mlflow:password@mysql:3306/mlflow --default-artifact-root=s3://mlflow/ --host=0.0.0.0 --port=5000"
+    environment:
+      MLFLOW_S3_ENDPOINT_URL: http://minio:9000
+      AWS_ACCESS_KEY_ID: minioadmin
+      AWS_SECRET_ACCESS_KEY: minioadmin
+    ports:
+      - "5000:5000"
+    networks:
+      - aws_mysql_test_nw
+    volumes:
+      - mlflow-aws-mysql-storage:/mlflow
+    depends_on:
+      - minio
+      - mysql
+
+volumes:
+  mysql-aws-storage:
+    driver: local
+  mlflow-aws-mysql-storage:
+    driver: local
+  minio-mysql-storage:
+    driver: local
+
+networks:
+  aws_mysql_test_nw:
+    driver: bridge
diff --git a/docker-compose.aws-test.yaml → docker-compose.aws-postgres-test.yaml b/docker-compose.aws-test.yaml → docker-compose.aws-postgres-test.yaml
@@ -2,7 +2,7 @@ version: "3.8"
 
 services:
   minio:
-    container_name: minio-test
+    container_name: minio-pg-test
     hostname: minio
     image: minio/minio:${MINIO_VERSION}
     entrypoint: sh
@@ -14,9 +14,9 @@ services:
       - "9000:9000"
       - "9001:9001"
     volumes:
-      - minio-storage:/data
+      - minio-pg-storage:/data
     networks:
-      - aws_test_nw
+      - aws_pg_test_nw
 
   postgres:
     image: "postgres:${POSTGRES_VERSION}"
@@ -29,10 +29,10 @@ services:
     volumes:
       - postgres-aws-storage:/var/lib/postgresql/data
     networks:
-      - aws_test_nw
+      - aws_pg_test_nw
 
   mlflow:
-    container_name: mlflow-aws-test
+    container_name: mlflow-aws-pg-test
     build:
       context: .
       dockerfile: Dockerfile
@@ -49,21 +49,21 @@ services:
     ports:
       - "5000:5000"
     networks:
-      - aws_test_nw
+      - aws_pg_test_nw
     volumes:
-      - mlflow-aws-storage:/mlflow
+      - mlflow-aws-pg-storage:/mlflow
     depends_on:
       - minio
       - postgres
 
 volumes:
   postgres-aws-storage:
     driver: local
-  mlflow-aws-storage:
+  mlflow-aws-pg-storage:
     driver: local
-  minio-storage:
+  minio-pg-storage:
     driver: local
 
 networks:
-  aws_test_nw:
+  aws_pg_test_nw:
     driver: bridge
diff --git a/docker-compose.azure-mysql-test.yaml b/docker-compose.azure-mysql-test.yaml
@@ -0,0 +1,69 @@
+version: "3.8"
+
+services:
+  azurite:
+    container_name: azurite-mysql-test
+    hostname: azurite
+    build:
+      context: test-containers/azurite
+      dockerfile: Dockerfile
+    command: sh -c "sh /run.sh | azurite -l /data --blobHost 0.0.0.0 --queueHost 0.0.0.0 --loose --skipApiVersionCheck"
+    environment:
+      AZURE_STORAGE_ACCOUNT: devstoreaccount1
+      AZURE_STORAGE_ACCESS_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+      COMMA_SEPARATED_CONTAINER_NAMES: "mlflow"
+    ports:
+      - "10000:10000"
+      - "10001:10001"
+    volumes:
+      - azurite-mysql-storage:/data
+      - ./test-containers/azurite/create-container.js:/create-container.js
+      - ./test-containers/azurite/run.sh:/run.sh
+    networks:
+      - azure_mysql_test_nw
+
+  mysql:
+    image: "mysql:${MYSQL_VERSION}"
+    container_name: mlflow-azure-mysql-db
+    environment:
+      MYSQL_ROOT_PASSWORD: password
+      MYSQL_DATABASE: mlflow
+      MYSQL_USER: mlflow
+      MYSQL_PASSWORD: password
+      MYSQL_TCP_PORT: 3306
+    ports:
+      - "3306:3306"
+    volumes:
+      - mysql-azure-storage:/var/lib/mysql
+    networks:
+      - azure_mysql_test_nw
+
+  mlflow:
+    container_name: mlflow-azure-pg-test
+    build:
+      context: .
+      dockerfile: Dockerfile
+    command: "mlflow server --backend-store-uri=mysql+pymysql://mlflow:password@mysql:3306/mlflow --default-artifact-root=wasbs://mlflow@devstoreaccount1.blob.core.windows.net --host=0.0.0.0 --port=5000"
+    environment:
+      AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1
+    ports:
+      - "5000:5000"
+    networks:
+      - azure_mysql_test_nw
+    volumes:
+      - mlflow-azure-mysql-storage:/mlflow
+    depends_on:
+      - azurite
+      - mysql
+
+volumes:
+  mysql-azure-storage:
+    driver: local
+  mlflow-azure-mysql-storage:
+    driver: local
+  azurite-mysql-storage:
+    driver: local
+
+networks:
+  azure_mysql_test_nw:
+    driver: bridge
diff --git a/docker-compose.azure-test.yaml → docker-compose.azure-postgres-test.yaml b/docker-compose.azure-test.yaml → docker-compose.azure-postgres-test.yaml
@@ -2,7 +2,7 @@ version: "3.8"
 
 services:
   azurite:
-    container_name: azurite-test
+    container_name: azurite-pg-test
     hostname: azurite
     build:
       context: test-containers/azurite
@@ -16,11 +16,11 @@ services:
       - "10000:10000"
       - "10001:10001"
     volumes:
-      - azurite-storage:/data
+      - azurite-pg-storage:/data
       - ./test-containers/azurite/create-container.js:/create-container.js
       - ./test-containers/azurite/run.sh:/run.sh
     networks:
-      - azure_test_nw
+      - azure_pg_test_nw
 
   postgres:
     image: "postgres:${POSTGRES_VERSION}"
@@ -33,10 +33,10 @@ services:
     volumes:
       - postgres-azure-storage:/var/lib/postgresql/data
     networks:
-      - azure_test_nw
+      - azure_pg_test_nw
 
   mlflow:
-    container_name: mlflow-azure-test
+    container_name: mlflow-azure-pg-test
     build:
       context: .
       dockerfile: Dockerfile
@@ -51,21 +51,21 @@ services:
     ports:
       - "5000:5000"
     networks:
-      - azure_test_nw
+      - azure_pg_test_nw
     volumes:
-      - mlflow-azure-storage:/mlflow
+      - mlflow-azure-pg-storage:/mlflow
     depends_on:
       - azurite
       - postgres
 
 volumes:
   postgres-azure-storage:
     driver: local
-  mlflow-azure-storage:
+  mlflow-azure-pg-storage:
     driver: local
-  azurite-storage:
+  azurite-pg-storage:
     driver: local
 
 networks:
-  azure_test_nw:
+  azure_pg_test_nw:
     driver: bridge
diff --git a/docker-compose.gcp-mysql-test.yaml b/docker-compose.gcp-mysql-test.yaml
@@ -0,0 +1,61 @@
+version: "3.8"
+
+services:
+  gcs:
+    container_name: gcs-mysql-test
+    hostname: gcs
+    image: fsouza/fake-gcs-server:${FAKE_GCS_SERVER_VERSION}
+    entrypoint: sh -c "mkdir -p /data/mlflow && /bin/fake-gcs-server -data /data -scheme http"
+    ports:
+      - "4443:4443"
+    volumes:
+      - gcs-mysql-storage:/data
+    networks:
+      - gcp_mysql_test_nw
+
+  mysql:
+    image: "mysql:${MYSQL_VERSION}"
+    container_name: mlflow-azure-mysql-db
+    environment:
+      MYSQL_ROOT_PASSWORD: password
+      MYSQL_DATABASE: mlflow
+      MYSQL_USER: mlflow
+      MYSQL_PASSWORD: password
+      MYSQL_TCP_PORT: 3306
+    ports:
+      - "3306:3306"
+    volumes:
+      - mysql-gcp-storage:/var/lib/mysql
+    networks:
+      - gcp_mysql_test_nw
+
+  mlflow:
+    container_name: mlflow-gcp-mysql-test
+    build:
+      context: .
+      dockerfile: Dockerfile
+    command: "mlflow server --backend-store-uri=mysql+pymysql://mlflow:password@mysql:3306/mlflow --default-artifact-root=gs://mlflow/ --host=0.0.0.0 --port=5000"
+    environment:
+      GOOGLE_CLOUD_PROJECT: mlflow
+      STORAGE_EMULATOR_HOST: http://gcs:4443
+    ports:
+      - "5000:5000"
+    networks:
+      - gcp_mysql_test_nw
+    volumes:
+      - mlflow-gcp-mysql-storage:/mlflow
+    depends_on:
+      - gcs
+      - mysql
+
+volumes:
+  mysql-gcp-storage:
+    driver: local
+  mlflow-gcp-mysql-storage:
+    driver: local
+  gcs-mysql-storage:
+    driver: local
+
+networks:
+  gcp_mysql_test_nw:
+    driver: bridge
diff --git a/docker-compose.gcp-test.yaml → docker-compose.gcp-postgres-test.yaml b/docker-compose.gcp-test.yaml → docker-compose.gcp-postgres-test.yaml
@@ -2,16 +2,16 @@ version: "3.8"
 
 services:
   gcs:
-    container_name: gcs-test
+    container_name: gcs-pg-test
     hostname: gcs
     image: fsouza/fake-gcs-server:${FAKE_GCS_SERVER_VERSION}
     entrypoint: sh -c "mkdir -p /data/mlflow && /bin/fake-gcs-server -data /data -scheme http"
     ports:
       - "4443:4443"
     volumes:
-      - gcs-storage:/data
+      - gcs-pg-storage:/data
     networks:
-      - gcp_test_nw
+      - gcp_pg_test_nw
 
   postgres:
     image: "postgres:${POSTGRES_VERSION}"
@@ -24,10 +24,10 @@ services:
     volumes:
       - postgres-gcp-storage:/var/lib/postgresql/data
     networks:
-      - gcp_test_nw
+      - gcp_pg_test_nw
 
   mlflow:
-    container_name: mlflow-gcp-test
+    container_name: mlflow-gcp-pg-test
     build:
       context: .
       dockerfile: Dockerfile
@@ -43,21 +43,21 @@ services:
     ports:
       - "5000:5000"
     networks:
-      - gcp_test_nw
+      - gcp_pg_test_nw
     volumes:
-      - mlflow-gcp-storage:/mlflow
+      - mlflow-gcp-pg-storage:/mlflow
     depends_on:
       - gcs
       - postgres
 
 volumes:
   postgres-gcp-storage:
     driver: local
-  mlflow-gcp-storage:
+  mlflow-gcp-pg-storage:
     driver: local
-  gcs-storage:
+  gcs-pg-storage:
     driver: local
 
 networks:
-  gcp_test_nw:
+  gcp_pg_test_nw:
     driver: bridge