chore(sipi): add comments (#1864)

Author: subotic

WORKSPACE

@@ -3,7 +3,10 @@ workspace(name = "io_dasch_dsp_api")
 # load http_archive method
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
 
-# bazel-skylib 1.0.2 released 2019.10.09 (https://github.com/bazelbuild/bazel-skylib/releases/tag/1.0.2)
+#####################################
+# Skylib                            #
+#####################################
+# 1.0.2 released 2019.10.09 (https://github.com/bazelbuild/bazel-skylib/releases/tag/1.0.2)
 skylib_version = "1.0.2"
 
 http_archive(
@@ -13,7 +16,82 @@ http_archive(
     url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel-skylib-{}.tar.gz".format(skylib_version, skylib_version),
 )
 
-# download rules_scala repository
+#####################################
+# Docker                            #
+#####################################
+
+rules_docker_version = "0.17.0"
+
+rules_docker_version_sha256 = "59d5b42ac315e7eadffa944e86e90c2990110a1c8075f1cd145f487e999d22b3"
+
+http_archive(
+    name = "io_bazel_rules_docker",
+    sha256 = rules_docker_version_sha256,
+    strip_prefix = "rules_docker-%s" % rules_docker_version,
+    url = "https://github.com/bazelbuild/rules_docker/releases/download/v%s/rules_docker-v%s.tar.gz" % (rules_docker_version, rules_docker_version),
+)
+
+load(
+    "@io_bazel_rules_docker//repositories:repositories.bzl",
+    container_repositories = "repositories",
+)
+
+container_repositories()
+
+load("@io_bazel_rules_docker//repositories:deps.bzl", container_deps = "deps")
+
+container_deps()
+
+# load container_pull method
+load(
+    "@io_bazel_rules_docker//container:container.bzl",
+    "container_pull",
+)
+
+# get distroless java
+container_pull(
+    name = "java_base",
+    # 'tag' is also supported, but digest is encouraged for reproducibility.
+    digest = "sha256:deadbeef",
+    registry = "gcr.io",
+    repository = "distroless/java",
+)
+
+# get openjdk
+container_pull(
+    name = "openjdk11",
+    digest = "sha256:0e51b455654bd162c485a6a6b5b120cc82db453d9265cc90f0c4fb5d14e2f62e",
+    registry = "docker.io",
+    repository = "adoptopenjdk",
+    tag = "11-jre-hotspot-bionic",
+)
+
+# get sipi
+load("//third_party:versions.bzl", "SIPI_IMAGE_DIGEST", "SIPI_REPOSITORY", "SIPI_VERSION")
+
+container_pull(
+    name = "sipi",
+    digest = SIPI_IMAGE_DIGEST,
+    registry = "docker.io",
+    repository = SIPI_REPOSITORY,
+    tag = SIPI_VERSION,
+)
+
+# get fuseki
+load("//third_party:versions.bzl", "FUSEKI_IMAGE_DIGEST", "FUSEKI_REPOSITORY", "FUSEKI_VERSION")
+
+container_pull(
+    name = "jenafuseki",
+    digest = FUSEKI_IMAGE_DIGEST,
+    registry = "docker.io",
+    repository = FUSEKI_REPOSITORY,
+    tag = FUSEKI_VERSION,
+)
+
+#####################################
+# Scala                             #
+#####################################
+
 rules_scala_version = "0ac75d3a044b8e316d1b11b90a7d044685bd72e8"  # 22.04.2021
 
 rules_scala_version_sha256 = "7624c95c19b60df943dbde90c54d09ecad9aca9432b1211da8352f131776ac36"
@@ -52,9 +130,10 @@ scalatest_repositories()
 
 scalatest_toolchain()
 
-#
-# Download the protobuf repository (needed by go and rules_scala_annex)
-#
+#####################################
+# Protobuf (Scala Annex)            #
+#####################################
+
 protobuf_tag = "3.12.3"
 
 protobuf_sha256 = "e5265d552e12c1f39c72842fa91d84941726026fa056d914ea6a25cd58d7bbf8"
@@ -71,8 +150,10 @@ load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
 
 protobuf_deps()
 
+#####################################
+# JAR Dependencies                  #
+#####################################
 #
-# download rules_jvm_external used for maven dependency resolution
 # defined in the third_party sub-folder
 #
 rules_jvm_external_version = "4.0"  # 6.01.2021
@@ -97,9 +178,9 @@ load("@maven//:defs.bzl", "pinned_maven_install")
 
 pinned_maven_install()
 
-#
-# Load rules_scala_annex, required by rules_twirl
-#
+#####################################
+# Scala Annex (Twirl dependency)    #
+#####################################
 rules_scala_annex_version = "2503b72a166610c14170b117c51033b42a32e48b"  # 29.06.2020
 
 rules_scala_annex_sha256 = "52d677dc8205db25a49824aade45984e3ef1b79c3bf761efede35d921033c3a4"
@@ -147,9 +228,9 @@ bind(
     actual = "@rules_scala_annex//src/main/scala:zinc_2_12_10",
 )
 
-#
-# download the rules_twirl repository (needed to compile twirl templates)
-#
+#####################################
+# Twirl templates                   #
+#####################################
 rules_twirl_version = "35389750d178f17f7ddd85b9335f7b8b8d662f78"  # 29.04.2020
 
 rules_twirl_version_sha256 = "d072049d0917b87e1eb677a4255509a7133ca71fc21c8de4b4536ca030eb3d3a"
@@ -170,9 +251,9 @@ load("@twirl//:defs.bzl", twirl_pinned_maven_install = "pinned_maven_install")
 
 twirl_pinned_maven_install()
 
-#
-# Download the rules_go repository
-#
+#####################################
+# Buildifier                        #
+#####################################
 # buildifier is written in Go and hence needs rules_go to be built.
 # See https://github.com/bazelbuild/rules_go for the up to date setup instructions.
 http_archive(
@@ -208,90 +289,11 @@ http_archive(
     url = "https://github.com/bazelbuild/buildtools/archive/master.zip",
 )
 
-# legacy variant used by rules_docker. Remove after rules_docker was updated to
-# newest rules_python
-load("@rules_python//python:pip.bzl", "pip_import", "pip_repositories")
-
-pip_repositories()
-
-#
-# Download the rules_docker repository at release v0.14.4
-#
-rules_docker_version = "0.14.4"
-
-rules_docker_version_sha256 = "4521794f0fba2e20f3bf15846ab5e01d5332e587e9ce81629c7f96c793bb7036"
-
-http_archive(
-    name = "io_bazel_rules_docker",
-    sha256 = rules_docker_version_sha256,
-    strip_prefix = "rules_docker-%s" % rules_docker_version,
-    url = "https://github.com/bazelbuild/rules_docker/releases/download/v%s/rules_docker-v%s.tar.gz" % (rules_docker_version, rules_docker_version),
-)
-
-load(
-    "@io_bazel_rules_docker//repositories:repositories.bzl",
-    container_repositories = "repositories",
-)
-
-container_repositories()
-
-load("@io_bazel_rules_docker//repositories:deps.bzl", container_deps = "deps")
-
-container_deps()
 
-load("@io_bazel_rules_docker//repositories:pip_repositories.bzl", "pip_deps")
 
-pip_deps()
-
-# load container_pull method
-load(
-    "@io_bazel_rules_docker//container:container.bzl",
-    "container_pull",
-)
-
-# get distroless java
-container_pull(
-    name = "java_base",
-    # 'tag' is also supported, but digest is encouraged for reproducibility.
-    digest = "sha256:deadbeef",
-    registry = "gcr.io",
-    repository = "distroless/java",
-)
-
-# get openjdk
-container_pull(
-    name = "openjdk11",
-    digest = "sha256:0e51b455654bd162c485a6a6b5b120cc82db453d9265cc90f0c4fb5d14e2f62e",
-    registry = "docker.io",
-    repository = "adoptopenjdk",
-    tag = "11-jre-hotspot-bionic",
-)
-
-# get sipi
-load("//third_party:versions.bzl", "SIPI_IMAGE_DIGEST", "SIPI_REPOSITORY", "SIPI_VERSION")
-
-container_pull(
-    name = "sipi",
-    digest = SIPI_IMAGE_DIGEST,
-    registry = "docker.io",
-    repository = SIPI_REPOSITORY,
-    tag = SIPI_VERSION,
-)
-
-# get fuseki
-load("//third_party:versions.bzl", "FUSEKI_IMAGE_DIGEST", "FUSEKI_REPOSITORY", "FUSEKI_VERSION")
-
-container_pull(
-    name = "jenafuseki",
-    digest = FUSEKI_IMAGE_DIGEST,
-    registry = "docker.io",
-    repository = FUSEKI_REPOSITORY,
-    tag = FUSEKI_VERSION,
-)
-
-#
-# download rules_pkg - basic packaging rules
-#
+#####################################
+# rules_pkg - basic packaging rules #
+#####################################
 rules_package_version = "0.2.4"
 
 rules_package_version_sha256 = "4ba8f4ab0ff85f2484287ab06c0d871dcb31cc54d439457d28fd4ae14b18450a"
@@ -302,14 +304,13 @@ http_archive(
     url = "https://github.com/bazelbuild/rules_pkg/releases/download/%s/rules_pkg-%s.tar.gz" % (rules_package_version, rules_package_version),
 )
 
-# load further dependencies of this rule
 load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
 
 rules_pkg_dependencies()
 
-#
-# download rules_stamp - stamping helper
-#
+#####################################
+# rules_stamp - stamping helper     #
+#####################################
 http_archive(
     name = "ecosia_rules_stamp",
     sha256 = "36d7ea381bfb2520f9353299b162434b25c77365d3c9e9459195c536da5e837d",

sipi/scripts/BUILD.bazel

@@ -4,7 +4,29 @@ load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
 
 filegroup(
     name = "scripts",
-    srcs = glob(["*.lua"]),
+    srcs = [
+        "admin_upload.lua",
+        "cache.lua",
+        "clean_temp_dir.lua",
+        "debug.lua",
+        "delete_temp_file.lua",
+        "exit.lua",
+        "file_info.lua",
+        "get_knora_session.lua",
+        "jwt.lua",
+        "send_response.lua",
+        "sipi.init-knora.lua",
+        "sipi.init-knora-no-auth.lua",
+        "sipi.init-knora-test.lua",
+        "store.lua",
+        "test1.lua",
+        "test2.lua",
+        "test_file_info.lua",
+        "test_functions.lua",
+        "test_knora_session_cookie.lua",
+        "upload.lua",
+        "util.lua",
+    ],
 )
 
 pkg_tar(

sipi/scripts/send_response.lua

@@ -49,20 +49,15 @@ function send_error(status, msg)
         msg_str = "Unknown error. Please report this as a possible bug in a Sipi route."
     end
 
-    local result
-
-    -- If this is an internal server error, log the message, and return a generic message to the client.
+    -- If this is an internal server error, log the message.
     if http_status // 100 == 5 then
         server.log(msg_str, server.loglevel.LOG_ERR)
-
-        result = {
-            message = "Internal server error"
-        }
-    else
-        result = {
-            message = msg_str
-        }
     end
+
+    local result
+    result = {
+        message = msg_str
+    }
 
     success, error_msg = server.sendHeader("Content-Type", "application/json")
     if not success then

sipi/scripts/upload.lua

@@ -38,7 +38,8 @@ end
 
 
 -- Buffer the response (helps with error handling).
-local success, error_msg = server.setBuffer()
+local success, error_msg
+success, error_msg = server.setBuffer()
 if not success then
     send_error(500, "server.setBuffer() failed: " .. error_msg)
     return
@@ -54,12 +55,14 @@ end
 local tmpFolder = config.imgroot .. '/tmp/'
 local exists
 success, exists = server.fs.exists(tmpFolder)
-if not success then
+if not success then -- tests server.fs.exists
+    -- fs.exist was not run successful. This does not mean, that the tmp folder is not there.
     send_error(500, "server.fs.exists() failed: " .. exists)
     return
 end
-if not exists then
-    local error_msg
+if not exists then -- checks the response of server.fs.exists
+    -- tmp folder does not exist
+    server.log("temp folder missing: " .. tmpFolder, server.loglevel.LOG_ERR)
     success, error_msg = server.fs.mkdir(tmpFolder, 511)
     if not success then
         send_error(500, "server.fs.mkdir() failed: " .. error_msg)

webapi/src/test/resources/logback-test.xml

@@ -51,6 +51,8 @@
     <logger name="kamon.metric" level="ERROR"/>
     <logger name="org.apache.http" level="INFO"/>
     <logger name="org.apache.http.wire" level="INFO"/>
+    <logger name="org.testcontainers" level="INFO"/>
+    <logger name="com.github.dockerjava" level="WARN"/>
 
     <!-- App -->
     <logger name="org.knora.webapi.app.ApplicationActor" level="INFO"/>

webapi/src/test/scala/org/knora/webapi/ITKnoraLiveSpec.scala

@@ -244,9 +244,8 @@ class ITKnoraLiveSpec(_system: ActorSystem)
     val sipiRequest = Post(s"$baseInternalSipiUrl/upload?token=$loginToken", sipiFormData)
 
     val sipiUploadResponseJson: JsObject = getResponseJson(sipiRequest)
-    // println(sipiUploadResponseJson.prettyPrint)
+
     val sipiUploadResponse: SipiUploadResponse = sipiUploadResponseJson.convertTo[SipiUploadResponse]
-    // println(s"sipiUploadResponse: $sipiUploadResponse")
 
     // Request the temporary file from Sipi.
     for (responseEntry <- sipiUploadResponse.uploadedFiles) {

webapi/src/test/scala/org/knora/webapi/TestContainers.scala

@@ -25,7 +25,7 @@ import com.typesafe.config.{Config, ConfigFactory}
 import org.testcontainers.containers.{BindMode, GenericContainer}
 import org.testcontainers.utility.DockerImageName
 
-import scala.collection.JavaConverters._
+import scala.jdk.CollectionConverters._
 
 /**
   * Provides all containers necessary for running tests.
@@ -69,7 +69,6 @@ object TestContainers {
   RedisContainer.withExposedPorts(6379)
   RedisContainer.start()
 
-  import scala.collection.JavaConverters._
   private val portMap = Map(
     "app.triplestore.fuseki.port" -> FusekiContainer.getFirstMappedPort,
     "app.sipi.internal-host" -> sipiIp,