feat(upgrade): Update repository on startup (#1643)

.github/workflows/main.yml

@@ -232,36 +232,6 @@ jobs:
           df -h
           docker system df
 
-  upgrade-unit-tests:
-    name: Upgrade Unit Tests
-    needs: compile
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-        with:
-          fetch-depth: 1
-      - name: cache maven artifacts
-        uses: actions/cache@v1
-        with:
-          path: ~/.ivy2
-          key: ${{ runner.OS }}-build-${{ hashFiles('**/KnoraBuild.sbt') }}
-          restore-keys: |
-            ${{ runner.OS }}-build-${{ env.cache-name }}-
-            ${{ runner.OS }}-build-
-            ${{ runner.OS }}-
-      - name: Disk Free
-        run: |
-          df -h
-          docker system df
-          docker system prune --all --force --volumes
-          df -h
-      - name: run upgrade tests
-        run: sbt "upgrade/test"
-      - name: Disk Free After
-        run: |
-          df -h
-          docker system df
-
   upgrade-integration-tests:
     name: Upgrade Integration Tests
     needs: compile
@@ -297,32 +267,8 @@ jobs:
           docker system df
           docker system prune --all --force --volumes
           df -h
-      - name: build all docker images
-        run: |
-          make build-all-scala
-          make build-all-images
-      - name: start and initialize graphdb-se
-        run: |
-          docker network create knora-net
-          docker run --rm -d --name=db --network=knora-net -p 127.0.0.1:7200:7200 -e GDB_HEAP_SIZE=$GDB_HEAP_SIZE daschswiss/knora-graphdb-se -Dgraphdb.license.file=/graphdb/graphdb.license
-          sleep 5
-          make init-db-test-minimal
-      - name: unzip test data
-        run: unzip $GITHUB_WORKSPACE/test-data/v7.0.0/v7.0.0-knora-test.trig.zip -d $GITHUB_WORKSPACE/test-data/v7.0.0/
-      - name: empty repository
-        run: $GITHUB_WORKSPACE/upgrade/graphdb-se/empty-repository.sh -r knora-test -u gaga -p gaga -h localhost:7200
-      - name: load v7.0.0 data
-        run: $GITHUB_WORKSPACE/upgrade/graphdb-se/upload-repository.sh -r knora-test -u gaga -p gaga -h localhost:7200 $GITHUB_WORKSPACE/test-data/v7.0.0/v7.0.0-knora-test.trig
-      - name: dump repository data
-        run: |
-          mkdir -p $GITHUB_WORKSPACE/upgrade/out
-          $GITHUB_WORKSPACE/upgrade/graphdb-se/dump-repository.sh -r knora-test -u gaga -p gaga -h localhost:7200 $GITHUB_WORKSPACE/upgrade/out/dump.trig
-      - name: run upgrade from inside docker
-        run: docker run --rm --network=knora-net -v $GITHUB_WORKSPACE/upgrade/out:/upgrade/out daschswiss/knora-upgrade /upgrade/out/dump.trig /upgrade/out/dump-upgraded.trig
-      - name: empty repository
-        run: $GITHUB_WORKSPACE/upgrade/graphdb-se/empty-repository.sh -r knora-test -u gaga -p gaga -h localhost:7200
-      - name: load upgraded data
-        run: $GITHUB_WORKSPACE/upgrade/graphdb-se/upload-repository.sh -r knora-test -u gaga -p gaga -h localhost:7200 $GITHUB_WORKSPACE/upgraded/out/dump-upgraded.trig
+      - name: test repository update
+        run: cd $GITHUB_WORKSPACE && make test-repository-update
       - name: Disk Free After
         run: |
           df -h
@@ -364,7 +310,13 @@ jobs:
   # publish only for develop and tags
   publish:
     name: Publish to Dockerhub
-    needs: [api-unit-tests, api-e2e-tests, api-integration-tests, upgrade-unit-tests, upgrade-integration-tests, docs-build-test]
+    needs: [
+      api-unit-tests,
+      api-e2e-tests,
+      api-integration-tests,
+      upgrade-integration-tests,
+      docs-build-test
+    ]
     runs-on: ubuntu-latest
     if: github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags')
     steps:

KnoraBuild.sbt

@@ -5,6 +5,7 @@ import com.typesafe.sbt.packager.docker.{Cmd, ExecCmd}
 import org.knora.Dependencies
 import sbt.Keys.version
 
+import scala.language.postfixOps
 import scala.sys.process.Process
 
 //////////////////////////////////////
@@ -226,6 +227,7 @@ lazy val knoraGraphDbSe: Project = knoraModule("knora-graphdb-se")
       )
   )
 
+
 //////////////////////////////////////
 // Knora's custom GraphDB-Free
 //////////////////////////////////////
@@ -265,6 +267,7 @@ lazy val knoraGraphdbFree: Project = knoraModule("knora-graphdb-free")
       )
   )
 
+
 //////////////////////////////////////
 // Knora's custom Sipi
 //////////////////////////////////////
@@ -343,94 +346,6 @@ lazy val knoraAssets: Project = knoraModule("knora-assets")
       )
   )
 
-//////////////////////////////////////
-// Knora upgrade scripts
-//////////////////////////////////////
-
-lazy val upgradeCommonSettings = Seq(
-    name := "upgrade"
-)
-
-lazy val upgrade: Project = knoraModule("upgrade")
-  .dependsOn(webapi)
-  .enablePlugins(JavaAppPackaging, DockerPlugin)
-  .settings(
-      upgradeCommonSettings,
-      Dependencies.upgradeLibraryDependencies,
-      // use jars (and not class directory) for run, test, console
-      exportJars := true,
-      unmanagedResourceDirectories in Compile += (rootBaseDir.value / "knora-ontologies"),
-
-      // add content of knora-ontologies to jar
-      mappings in (Compile, packageBin) ++= Seq (
-          (rootBaseDir.value / "knora-ontologies" / "knora-admin.ttl") -> "knora-ontologies/knora-admin.ttl",
-          (rootBaseDir.value / "knora-ontologies" / "knora-base.ttl") -> "knora-ontologies/knora-base.ttl",
-          (rootBaseDir.value / "knora-ontologies" / "salsah-gui.ttl") -> "knora-ontologies/salsah-gui.ttl",
-          (rootBaseDir.value / "knora-ontologies" / "standoff-data.ttl") -> "knora-ontologies/standoff-data.ttl",
-          (rootBaseDir.value / "knora-ontologies" / "standoff-onto.ttl") -> "knora-ontologies/standoff-onto.ttl",
-      ),
-      // contentOf("salsah1/src/main/resources").toMap.mapValues("config/" + _)
-      // (rootBaseDir.value / "knora-ontologies") -> "knora-ontologies",
-
-  )
-  .settings(
-      scalacOptions ++= Seq("-feature", "-unchecked", "-deprecation", "-Yresolve-term-conflict:package"),
-      logLevel := Level.Info,
-      run / fork := true,
-      run / javaOptions ++= upgradeJavaRunOptions,
-      Compile / run / mainClass := Some("org.knora.upgrade.Main"),
-      Test / fork := true,
-      Test / javaOptions ++= upgradeJavaTestOptions,
-      Test / parallelExecution := false,
-      /* show full stack traces and test case durations */
-      Test / testOptions += Tests.Argument("-oDF"),
-  )
-  .settings(
-      // Skip packageDoc and packageSrc task on stage
-      Compile / packageDoc / mappings := Seq(),
-      Compile / packageSrc / mappings := Seq(),
-
-      Universal / mappings ++= {
-          // copy the different folders
-          directory("upgrade/graphdb-se")
-      },
-
-      // add dockerCommands used to create the image
-      // docker:stage, docker:publishLocal, docker:publish, docker:clean
-      dockerRepository := Some("dhlabbasel"),
-      dockerUpdateLatest := true,
-      maintainer := "400790+subotic@users.noreply.github.com",
-      Docker / dockerCommands := Seq(
-          Cmd("FROM", "adoptopenjdk/openjdk11:alpine-jre"),
-          Cmd("LABEL", s"""MAINTAINER="${maintainer.value}""""),
-          Cmd("RUN apk update && apk upgrade && apk add bash"),
-
-          Cmd("ENV", """KNORA_UPGRADE_DOCKER="true""""),
-          Cmd("COPY", "opt/docker", "/upgrade"),
-          Cmd("WORKDIR", "/upgrade"),
-          ExecCmd("ENTRYPOINT", "/upgrade/bin/upgrade"),
-      ),
-  )
-
-lazy val upgradeJavaRunOptions = Seq(
-    // "-showversion",
-    "-Xms1G",
-    "-Xmx1G"
-    // "-verbose:gc",
-    //"-XX:+UseG1GC",
-    //"-XX:MaxGCPauseMillis=500",
-    //"-XX:MaxMetaspaceSize=4096m"
-)
-
-lazy val upgradeJavaTestOptions = Seq(
-    // "-showversion",
-    "-Xms1G",
-    "-Xmx1G"
-    // "-verbose:gc",
-    //"-XX:+UseG1GC",
-    //"-XX:MaxGCPauseMillis=500",
-    //"-XX:MaxMetaspaceSize=4096m"
-)
 
 //////////////////////////////////////
 // SALSAH1 (./salsah1)
@@ -533,13 +448,12 @@ lazy val javaTestOptions = Seq(
 )
 
 
-lazy val HeadlessTest = config("headless") extend (Test)
+lazy val HeadlessTest = config("headless") extend Test
 lazy val javaHeadlessTestOptions = Seq(
     "-Dconfig.resource=headless-testing.conf"
 ) ++ javaTestOptions
 
 
-
 //////////////////////////////////////
 // WEBAPI (./webapi)
 //////////////////////////////////////
@@ -601,6 +515,23 @@ lazy val webapi = knoraModule("webapi")
             inConfig(FusekiIt)(Defaults.testTasks ++ Seq(forkOptions := Defaults.forkOptionsTask.value)),
             inConfig(EmbeddedJenaTDBTest)(Defaults.testTasks ++ Seq(forkOptions := Defaults.forkOptionsTask.value))
         )
+
+        .settings(
+          exportJars := true,
+          unmanagedResourceDirectories in Compile += (rootBaseDir.value / "knora-ontologies"),
+
+          // add content of knora-ontologies to jar
+          mappings in (Compile, packageBin) ++= Seq (
+              (rootBaseDir.value / "knora-ontologies" / "knora-admin.ttl") -> "knora-ontologies/knora-admin.ttl",
+              (rootBaseDir.value / "knora-ontologies" / "knora-base.ttl") -> "knora-ontologies/knora-base.ttl",
+              (rootBaseDir.value / "knora-ontologies" / "salsah-gui.ttl") -> "knora-ontologies/salsah-gui.ttl",
+              (rootBaseDir.value / "knora-ontologies" / "standoff-data.ttl") -> "knora-ontologies/standoff-data.ttl",
+              (rootBaseDir.value / "knora-ontologies" / "standoff-onto.ttl") -> "knora-ontologies/standoff-onto.ttl",
+          ),
+          // contentOf("salsah1/src/main/resources").toMap.mapValues("config/" + _)
+          // (rootBaseDir.value / "knora-ontologies") -> "knora-ontologies",
+        )
+
         .settings(
             scalacOptions ++= Seq("-feature", "-unchecked", "-deprecation", "-Yresolve-term-conflict:package"),
 

Makefile

@@ -24,7 +24,7 @@ docs-build: ## build the docs
 
 .PHONY: build-all-scala
 build-all-scala: ## build all scala projects
-	@sbt webapi/universal:stage knora-graphdb-se/universal:stage knora-graphdb-free/universal:stage knora-sipi/universal:stage salsah1/universal:stage upgrade/universal:stage knora-assets/universal:stage webapi_test/universal:stage webapi_it/universal:stage
+	@sbt webapi/universal:stage knora-graphdb-se/universal:stage knora-graphdb-free/universal:stage knora-sipi/universal:stage salsah1/universal:stage knora-assets/universal:stage webapi_test/universal:stage webapi_it/universal:stage
 
 ## knora-api
 .PHONY: build-knora-api-image
@@ -83,15 +83,6 @@ build-knora-salsah1-image: build-all-scala ## build and publish knora-salsah1 do
 publish-knora-salsah1-image: build-knora-salsah1-image ## publish knora-salsah1 image to Dockerhub
 	docker push $(KNORA_SALSAH1_IMAGE)
 
-## knora-upgrade
-.PHONY: build-knora-upgrade-image
-build-knora-upgrade-image: build-all-scala ## build and publish knora-upgrade docker image locally
-	docker build -t $(KNORA_UPGRADE_IMAGE) -t $(REPO_PREFIX)/$(KNORA_UPGRADE_REPO):latest -f docker/knora-upgrade.dockerfile  upgrade/target/universal
-
-.PHONY: publish-knora-upgrade-image
-publish-knora-upgrade-image: build-knora-upgrade-image ## publish knora-upgrade image to Dockerhub
-	docker push $(KNORA_UPGRADE_IMAGE)
-
 ## knora-assets
 .PHONY: build-knora-assets-image
 build-knora-assets-image: build-all-scala ## build and publish knora-assets docker image locally
@@ -103,10 +94,10 @@ publish-knora-assets-image: build-knora-assets-image ## publish knora-assets ima
 
 ## all images
 .PHONY: build-all-images
-build-all-images: build-knora-api-image build-knora-graphdb-se-image build-knora-graphdb-free-image build-knora-sipi-image build-knora-salsah1-image build-knora-upgrade-image build-knora-assets-image  ## build all Docker images
+build-all-images: build-knora-api-image build-knora-graphdb-se-image build-knora-graphdb-free-image build-knora-sipi-image build-knora-salsah1-image build-knora-assets-image  ## build all Docker images
 
 .PHONY: publish-all-images
-publish-all-images: publish-knora-api-image publish-knora-graphdb-se-image publish-knora-graphdb-free-image publish-knora-sipi-image publish-knora-salsah1-image publish-knora-upgrade-image publish-knora-assets-image ## publish all Docker images
+publish-all-images: publish-knora-api-image publish-knora-graphdb-se-image publish-knora-graphdb-free-image publish-knora-sipi-image publish-knora-salsah1-image publish-knora-assets-image ## publish all Docker images
 
 #################################
 ## Docker-Compose targets
@@ -164,6 +155,7 @@ stack-restart: stack-up ## re-starts the knora-stack: graphdb, sipi, redis, api,
 .PHONY: stack-restart-api
 stack-restart-api: ## re-starts the api. Usually used after loading data into GraphDB.
 	docker-compose -f docker/knora.docker-compose.yml restart api
+	@$(CURRENT_DIR)/webapi/scripts/wait-for-knora.sh
 
 .PHONY: stack-logs
 stack-logs: ## prints out and follows the logs of the running knora-stack.
@@ -343,12 +335,23 @@ test-js-lib-integration: clean-local-tmp stack-without-api ## run knora-api-js-l
 	@$(MAKE) -f $(THIS_FILE) init-db-test
 	@sleep 15
 	@$(MAKE) -f $(THIS_FILE) stack-restart-api
-	sleep 15
 	@$(MAKE) -f $(THIS_FILE) stack-logs-api-no-follow
 	@git clone --single-branch --depth 1 https://github.com/dasch-swiss/knora-api-js-lib.git $(CURRENT_DIR)/.tmp/js-lib
 	$(MAKE) -C $(CURRENT_DIR)/.tmp/js-lib npm-install
 	$(MAKE) -C $(CURRENT_DIR)/.tmp/js-lib test
 
+.PHONY: test-repository-update
+test-repository-update: stack-without-api
+	@sleep 15
+	@$(MAKE) -f $(THIS_FILE) init-db-test-minimal
+	@rm -rf /tmp/knora-test-data/v7.0.0/
+	@mkdir -p /tmp/knora-test-data/v7.0.0/
+	@unzip $(CURRENT_DIR)/test-data/v7.0.0/v7.0.0-knora-test.trig.zip -d /tmp/knora-test-data/v7.0.0/
+	$(CURRENT_DIR)/webapi/scripts/graphdb-empty-repository.sh -r knora-test -u gaga -p gaga -h localhost:7200
+	$(CURRENT_DIR)/webapi/scripts/graphdb-upload-repository.sh -r knora-test -u gaga -p gaga -h localhost:7200 /tmp/knora-test-data/v7.0.0/v7.0.0-knora-test.trig
+	@$(MAKE) -f $(THIS_FILE) stack-restart-api
+	@$(MAKE) -f $(THIS_FILE) stack-logs-api-no-follow
+
 .PHONY: init-db-test
 init-db-test: ## initializes the knora-test repository
 	@echo $@
@@ -413,7 +416,6 @@ info: ## print out all variables
 	@echo "KNORA_GRAPHDB_FREE_IMAGE: \t $(KNORA_GRAPHDB_FREE_IMAGE)"
 	@echo "KNORA_SIPI_IMAGE: \t\t $(KNORA_SIPI_IMAGE)"
 	@echo "KNORA_ASSETS_IMAGE: \t\t $(KNORA_ASSETS_IMAGE)"
-	@echo "KNORA_UPGRADE_IMAGE: \t\t $(KNORA_UPGRADE_IMAGE)"
 	@echo "KNORA_SALSAH1_IMAGE: \t\t $(KNORA_SALSAH1_IMAGE)"
 	@echo "KNORA_GDB_LICENSE: \t\t $(KNORA_GDB_LICENSE)"
 	@echo "KNORA_GDB_IMPORT: \t\t $(KNORA_GDB_IMPORT)"