fix: loading of data (DSP-445) (#1669)

dasch-swiss · Jul 2, 2020 · 3f8d406b0f44afc2d6928a44e103d613031c3f48 · 3f8d406
1 parent 2dc5361
commit 3f8d406b0f44afc2d6928a44e103d613031c3f48
diff --git a/KnoraBuild.sbt b/KnoraBuild.sbt
@@ -415,8 +415,7 @@ lazy val webapi = knoraModule("webapi")
           (rootBaseDir.value / "knora-ontologies" / "salsah-gui.ttl") -> "knora-ontologies/salsah-gui.ttl",
           (rootBaseDir.value / "knora-ontologies" / "standoff-data.ttl") -> "knora-ontologies/standoff-data.ttl",
           (rootBaseDir.value / "knora-ontologies" / "standoff-onto.ttl") -> "knora-ontologies/standoff-onto.ttl",
-          (rootBaseDir.value / "webapi" / "scripts" / "fuseki-knora-test-repository-config.ttl") -> "webapi/scripts/fuseki-knora-test-repository-config.ttl",
-          (rootBaseDir.value / "webapi" / "scripts" / "fuseki-knora-test-unit-repository-config.ttl") -> "webapi/scripts/fuseki-knora-test-unit-repository-config.ttl",
+          (rootBaseDir.value / "webapi" / "scripts" / "fuseki-repository-config.ttl.template") -> "webapi/scripts/fuseki-repository-config.ttl.template"
       ),
       // contentOf("salsah1/src/main/resources").toMap.mapValues("config/" + _)
       // (rootBaseDir.value / "knora-ontologies") -> "knora-ontologies",

diff --git a/Makefile b/Makefile
@@ -232,14 +232,12 @@ test: build-all-images ## runs all tests.
 	sbt webapi/test webapi/it:test
 
 .PHONY: test-repository-update
-test-repository-update: stack-down-delete-volumes stack-without-api
-	@sleep 15
-	@$(MAKE) -f $(THIS_FILE) init-db-test-minimal
+test-repository-update: init-db-test-minimal
 	@rm -rf /tmp/knora-test-data/v7.0.0/
 	@mkdir -p /tmp/knora-test-data/v7.0.0/
 	@unzip $(CURRENT_DIR)/test-data/v7.0.0/v7.0.0-knora-test.trig.zip -d /tmp/knora-test-data/v7.0.0/
-	$(CURRENT_DIR)/webapi/scripts/fuseki-empty-repository.sh -r knora-test -u gaga -p gaga -h localhost:3030
-	$(CURRENT_DIR)/webapi/scripts/fuseki-upload-repository.sh -r knora-test -u gaga -p gaga -h localhost:3030 /tmp/knora-test-data/v7.0.0/v7.0.0-knora-test.trig
+	$(CURRENT_DIR)/webapi/scripts/fuseki-empty-repository.sh -r knora-test -u admin -p test -h localhost:3030
+	$(CURRENT_DIR)/webapi/scripts/fuseki-upload-repository.sh -r knora-test -u admin -p test -h localhost:3030 /tmp/knora-test-data/v7.0.0/v7.0.0-knora-test.trig
 	@$(MAKE) -f $(THIS_FILE) stack-restart-api
 	@$(MAKE) -f $(THIS_FILE) stack-logs-api-no-follow
 

diff --git a/webapi/scripts/fuseki-ci-prepare.sh b/webapi/scripts/fuseki-ci-prepare.sh
diff --git a/webapi/scripts/fuseki-dump-repository.sh b/webapi/scripts/fuseki-dump-repository.sh
@@ -53,4 +53,4 @@ if [[ -z "${HOST}" ]]; then
     HOST="localhost:8080"
 fi
 
-curl -sS -X GET -H "Accept: application/trig" -u "${USERNAME}:${PASSWORD}" "http://${HOST}/fuseki/${REPOSITORY}" > "${FILE}"
+curl -sS -X GET -H "Accept: application/trig" -u "${USERNAME}:${PASSWORD}" "http://${HOST}/${REPOSITORY}" > "${FILE}"
diff --git a/webapi/scripts/fuseki-functions.sh b/webapi/scripts/fuseki-functions.sh
@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+
+#set -x
+
+POSITIONAL=()
+while [[ $# -gt 0 ]]; do
+  key="$1"
+
+  case $key in
+  -r | --repository)
+    REPOSITORY="$2"
+    shift # past argument
+    shift # past value
+    ;;
+  -u | --username)
+    USERNAME="$2"
+    shift # past argument
+    shift # past value
+    ;;
+  -p | --password)
+    PASSWORD="$2"
+    shift # past argument
+    shift # past value
+    ;;
+  -h | --host)
+    HOST="$2"
+    shift # past argument
+    shift # past value
+    ;;
+  *) # unknown option
+    POSITIONAL+=("$1") # save it in an array for later
+    shift              # past argument
+    ;;
+  esac
+done
+set -- "${POSITIONAL[@]}" # restore positional parameters
+
+FILE="$1"
+
+if [[ -z "${REPOSITORY}" ]]; then
+  REPOSITORY="knora-test"
+fi
+
+if [[ -z "${HOST}" ]]; then
+  HOST="localhost:3030"
+fi
+
+if [[ -z "${USERNAME}" ]]; then
+  USERNAME="admin"
+fi
+
+if [[ -z "${PASSWORD}" ]]; then
+  PASSWORD="test"
+fi
+
+delete-repository() {
+  STATUS=$(curl -s -o /dev/null -w '%{http_code}' -u ${USERNAME}:${PASSWORD} -X DELETE http://${HOST}/\$/datasets/${REPOSITORY})
+
+  if [ "${STATUS}" -eq 200 ]; then
+    echo "==> delete repository done"
+    return 0
+  else
+    echo "==> delete repository failed"
+    return 1
+  fi
+}
+
+create-repository() {
+  REPOSITORY_CONFIG=$(sed "s/@REPOSITORY@/${REPOSITORY}/g" ./fuseki-repository-config.ttl.template)
+  STATUS=$(curl -s -o /dev/null -w '%{http_code}' -u ${USERNAME}:${PASSWORD} -H "Content-Type:text/turtle; charset=utf-8" --data-raw "${REPOSITORY_CONFIG}" -X POST http://${HOST}/\$/datasets)
+
+  if [ "${STATUS}" -eq 200 ]; then
+    echo "==> create repository done"
+    return 0
+  else
+    echo "==> create repository failed"
+    return 1
+  fi
+}
+
+upload-graph() {
+  STATUS=$(curl -s -o /dev/null -w '%{http_code}' -u ${USERNAME}:${PASSWORD} -H "Content-Type:text/turtle; charset=utf-8" --data-binary @$1 -X PUT http://${HOST}/${REPOSITORY}\?graph\="$2")
+
+  if [ "${STATUS}" -eq 201 ]; then
+    echo "==> 201 Created: $1 -> $2"
+    return 0
+  elif [ "${STATUS}" -eq 200 ]; then
+    echo "==> 200 OK: $1 -> $2"
+    return 0
+  else
+    echo "==> failed with status code ${STATUS}: $1 -> $2"
+    return 1
+  fi
+}
diff --git a/webapi/scripts/fuseki-init-knora-test-minimal.sh b/webapi/scripts/fuseki-init-knora-test-minimal.sh
@@ -1,90 +1,11 @@
 #!/usr/bin/env bash
 
-#set -x
-
-POSITIONAL=()
-while [[ $# -gt 0 ]]; do
-  key="$1"
-
-  case $key in
-  -u | --username)
-    USERNAME="$2"
-    shift # past argument
-    shift # past value
-    ;;
-  -p | --password)
-    PASSWORD="$2"
-    shift # past argument
-    shift # past value
-    ;;
-  -h | --host)
-    HOST="$2"
-    shift # past argument
-    shift # past value
-    ;;
-  *) # unknown option
-    POSITIONAL+=("$1") # save it in an array for later
-    shift              # past argument
-    ;;
-  esac
-done
-set -- "${POSITIONAL[@]}" # restore positional parameters
-
-FILE="$1"
+# Including fuseki-funcions.sh implementing delete, create, and upload.
+source fuseki-functions.sh
 
+# Name of the repository
 REPOSITORY="knora-test"
 
-if [[ -z "${HOST}" ]]; then
-  HOST="localhost:3030"
-fi
-
-if [[ -z "${USERNAME}" ]]; then
-  USERNAME="admin"
-fi
-
-if [[ -z "${PASSWORD}" ]]; then
-  PASSWORD="test"
-fi
-
-delete-repository() {
-  STATUS=$(curl -s -o /dev/null -w '%{http_code}' -u ${USERNAME}:${PASSWORD} -X DELETE http://${HOST}/\$/datasets/${REPOSITORY})
-
-  if [ "${STATUS}" -eq 200 ]; then
-    echo "==> delete repository done"
-    return 0
-  else
-    echo "==> delete repository failed"
-    return 1
-  fi
-}
-
-create-repository() {
-  STATUS=$(curl -s -o /dev/null -w '%{http_code}' -u ${USERNAME}:${PASSWORD} -F data=@./fuseki-knora-test-repository-config.ttl http://${HOST}/\$/datasets)
-
-  if [ "${STATUS}" -eq 200 ]; then
-    echo "==> create repository done"
-    return 0
-  else
-    echo "==> create repository failed"
-    return 1
-  fi
-}
-
-upload-graph() {
-  STATUS=$(curl -s -o /dev/null -w '%{http_code}' -u ${USERNAME}:${PASSWORD} -H "Content-Type:text/turtle" -d @$1 -X PUT http://${HOST}/${REPOSITORY}\?graph\="$2")
-
-  if [ "${STATUS}" -eq 201 ]; then
-    echo "==> 201 Created: $1 -> $2"
-    return 0
-  elif [ "${STATUS}" -eq 200 ]; then
-    echo "==> 200 OK: $1 -> $2"
-    return 0
-  else
-    echo "==> failed with status code ${STATUS}: $1 -> $2"
-    return 1
-  fi
-}
-
 # delete-repository // delete dos not work correctly. need to delete database manually.
 create-repository
 upload-graph ../../knora-ontologies/knora-admin.ttl http://www.knora.org/ontology/knora-admin

diff --git a/webapi/scripts/fuseki-init-knora-test-unit-minimal.sh b/webapi/scripts/fuseki-init-knora-test-unit-minimal.sh
@@ -1,90 +1,11 @@
 #!/usr/bin/env bash
 
-#set -x
-
-POSITIONAL=()
-while [[ $# -gt 0 ]]; do
-  key="$1"
-
-  case $key in
-  -u | --username)
-    USERNAME="$2"
-    shift # past argument
-    shift # past value
-    ;;
-  -p | --password)
-    PASSWORD="$2"
-    shift # past argument
-    shift # past value
-    ;;
-  -h | --host)
-    HOST="$2"
-    shift # past argument
-    shift # past value
-    ;;
-  *) # unknown option
-    POSITIONAL+=("$1") # save it in an array for later
-    shift              # past argument
-    ;;
-  esac
-done
-set -- "${POSITIONAL[@]}" # restore positional parameters
-
-FILE="$1"
+# Including fuseki-funcions.sh implementing delete, create, and upload.
+source fuseki-functions.sh
 
+# Name of the repository
 REPOSITORY="knora-test-unit"
 
-if [[ -z "${HOST}" ]]; then
-  HOST="localhost:3030"
-fi
-
-if [[ -z "${USERNAME}" ]]; then
-  USERNAME="admin"
-fi
-
-if [[ -z "${PASSWORD}" ]]; then
-  PASSWORD="test"
-fi
-
-delete-repository() {
-  STATUS=$(curl -s -o /dev/null -w '%{http_code}' -u ${USERNAME}:${PASSWORD} -X DELETE http://${HOST}/\$/datasets/${REPOSITORY})
-
-  if [ "${STATUS}" -eq 200 ]; then
-    echo "==> delete repository done"
-    return 0
-  else
-    echo "==> delete repository failed"
-    return 1
-  fi
-}
-
-create-repository() {
-  STATUS=$(curl -s -o /dev/null -w '%{http_code}' -u ${USERNAME}:${PASSWORD} -F data=@./fuseki-knora-test-unit-repository-config.ttl http://${HOST}/\$/datasets)
-
-  if [ "${STATUS}" -eq 200 ]; then
-    echo "==> create repository done"
-    return 0
-  else
-    echo "==> create repository failed"
-    return 1
-  fi
-}
-
-upload-graph() {
-  STATUS=$(curl -s -o /dev/null -w '%{http_code}' -u ${USERNAME}:${PASSWORD} -H "Content-Type:text/turtle" -d @$1 -X PUT http://${HOST}/${REPOSITORY}\?graph\="$2")
-
-  if [ "${STATUS}" -eq 201 ]; then
-    echo "==> 201 Created: $1 -> $2"
-    return 0
-  elif [ "${STATUS}" -eq 200 ]; then
-    echo "==> 200 OK: $1 -> $2"
-    return 0
-  else
-    echo "==> failed with status code ${STATUS}: $1 -> $2"
-    return 1
-  fi
-}
-
 # delete-repository // delete dos not work correctly. need to delete database manually.
 create-repository
 upload-graph ../../knora-ontologies/knora-admin.ttl http://www.knora.org/ontology/knora-admin