[Deprecations] Deprecate SQLite Migrations (#5650)

.github/workflows/ci.yaml

@@ -117,6 +117,7 @@ jobs:
         echo "tag=$(echo $unstable_tag)" >> $GITHUB_OUTPUT
     - name: Run Dockerized tests
       run: MLRUN_DOCKER_REGISTRY=ghcr.io/ MLRUN_DOCKER_CACHE_FROM_TAG=${{ steps.docker_cache.outputs.tag }} make test-integration-dockerized
+      timeout-minutes: 90
 
   migrations-tests:
     name: Run Dockerized Migrations Tests

.gitignore

@@ -32,6 +32,7 @@ tests/project.yaml
 mlrun/utils/version/version.json
 tests/system/env.yml
 project.yaml
+server/api/db/mlrun.db
 
 # pyenv file for working with several python versions
 .python-version

Makefile

@@ -114,16 +114,12 @@ install-all-requirements: ## Install all requirements needed for development and
 	python -m pip install --upgrade $(MLRUN_PIP_NO_CACHE_FLAG) pip~=$(MLRUN_PIP_VERSION)
 	python -m pip install .[all]
 
-.PHONY: create-migration-sqlite
-create-migration-sqlite: ## Create a DB migration (MLRUN_MIGRATION_MESSAGE must be set)
-	./automation/scripts/create_migration_sqlite.sh
-
 .PHONY: create-migration-mysql
 create-migration-mysql: ## Create a DB migration (MLRUN_MIGRATION_MESSAGE must be set)
 	./automation/scripts/create_migration_mysql.sh
 
 .PHONY: create-migration
-create-migration: create-migration-sqlite create-migration-mysql
+create-migration: create-migration-mysql
 	@echo "Migrations created successfully"
 
 .PHONY: bump-version
@@ -481,19 +477,14 @@ test-migrations-dockerized: build-test ## Run mlrun db migrations tests in docke
 		-t \
 		--rm \
 		--network='host' \
+		-v $(shell pwd):/mlrun \
 		-v /tmp:/tmp \
 		-v /var/run/docker.sock:/var/run/docker.sock \
 		$(MLRUN_TEST_IMAGE_NAME_TAGGED) make test-migrations
 
 .PHONY: test-migrations
 test-migrations: clean ## Run mlrun db migrations tests
-	python -m pytest -v \
-		--capture=no \
-		--disable-warnings \
-		--durations=100 \
-		-rf \
-		--test-alembic \
-		server/api/migrations_sqlite/tests/*
+	./automation/scripts/test_migration_mysql.sh
 
 .PHONY: test-system-dockerized
 test-system-dockerized: build-test-system ## Run mlrun system tests in docker container

automation/scripts/create_migration_mysql.sh

@@ -35,13 +35,12 @@ export MLRUN_HTTPDB__DSN="mysql+pymysql://root:pass@localhost:3306/mlrun"
 docker run \
 	--name=migration-db \
 	--rm \
-	-v "${ROOT_DIR}:/mlrun" \
 	-p 3306:3306 \
 	-e MYSQL_ROOT_PASSWORD="pass" \
 	-e MYSQL_ROOT_HOST="%" \
 	-e MYSQL_DATABASE="mlrun" \
 	-d \
-	mysql/mysql-server:8.0 \
+	gcr.io/iguazio/mlrun-mysql:8.0 \
 	--character-set-server=utf8 \
 	--collation-server=utf8_bin
 
@@ -57,6 +56,6 @@ done
 
 export PYTHONPATH=$ROOT_DIR
 
-alembic -c "${ROOT_DIR}/server/api/alembic_mysql.ini" upgrade head
-alembic -c "${ROOT_DIR}/server/api/alembic_mysql.ini" revision --autogenerate -m "${MLRUN_MIGRATION_MESSAGE}"
-
+cd ${ROOT_DIR}/server/api
+alembic upgrade head
+alembic revision --autogenerate -m "${MLRUN_MIGRATION_MESSAGE}"

automation/scripts/create_migration_sqlite.sh → automation/scripts/test_migration_mysql.sh

@@ -16,24 +16,44 @@
 
 set -e
 
-if [ -z "$MLRUN_MIGRATION_MESSAGE" ]; then
-	echo "Environment variable MLRUN_MIGRATION_MESSAGE not set"
-	exit 1
-fi
-
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-export ROOT_DIR="${SCRIPT_DIR}/../.."
-
 function cleanup {
-	rm -f "${ROOT_DIR}/server/api/migrations_sqlite/mlrun.db"
+	docker kill migration-db
 }
 trap cleanup SIGHUP SIGINT SIGTERM EXIT
 
-export MLRUN_HTTPDB__DSN="sqlite:///${ROOT_DIR}/server/api/migrations_sqlite/mlrun.db?check_same_thread=false"
 
-cleanup
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+ROOT_DIR="${SCRIPT_DIR}/../.."
+
+export MLRUN_HTTPDB__DSN="mysql+pymysql://root:pass@localhost:3306/mlrun"
+
+docker run \
+	--name=migration-db \
+	--rm \
+	-p 3306:3306 \
+	-e MYSQL_ROOT_PASSWORD="pass" \
+	-e MYSQL_ROOT_HOST="%" \
+	-e MYSQL_DATABASE="mlrun" \
+	-d \
+	gcr.io/iguazio/mlrun-mysql:8.0 \
+	--character-set-server=utf8 \
+	--collation-server=utf8_bin
+
+times=0
+while ! docker exec migration-db mysql --user=root --password=pass -e "status" > /dev/null 2>&1; do
+	echo "Waiting for database connection..."
+	sleep 2
+	if [ $times -ge 60 ]; then
+		exit 1
+	fi
+	times=$(( times + 1 ))
+done
 
 export PYTHONPATH=$ROOT_DIR
 
-alembic -c "${ROOT_DIR}/server/api/alembic.ini" upgrade head
-alembic -c "${ROOT_DIR}/server/api/alembic.ini" revision --autogenerate -m "${MLRUN_MIGRATION_MESSAGE}"
+python -m pytest -v \
+		--capture=no \
+		--disable-warnings \
+		--durations=100 \
+		-rf \
+		${ROOT_DIR}/server/api/migrations/tests/*

server/api/alembic.ini

@@ -2,7 +2,7 @@
 
 [alembic]
 # path to migration scripts
-script_location = %(here)s/migrations_sqlite
+script_location = %(here)s/migrations
 
 # template used to generate migration files
 # file_template = %%(rev)s_%%(slug)s
@@ -36,7 +36,7 @@ script_location = %(here)s/migrations_sqlite
 # output_encoding = utf-8
 
 # this is overridden anyway in the alembic env.py by what ever is configured in mlrun.mlconf
-sqlalchemy.url = sqlite:///test.db
+sqlalchemy.url = mysql+pymysql://root:pass@localhost:3306/mlrun
 
 
 [post_write_hooks]

server/api/db/sqldb/db.py

@@ -88,8 +88,10 @@
     Run,
     Schedule,
     User,
+    _labeled,
+    _tagged,
+    _with_notifications,
 )
-from server.api.db.sqldb.models.common import _labeled, _tagged, _with_notifications
 
 NULL = None  # Avoid flake8 issuing warnings when comparing in filter
 unversioned_tagged_object_uid_prefix = "unversioned-"