PR for #68 - Add a Security Domain element to the SystemComplexType

.github/workflows/build_and_release.yml

@@ -68,7 +68,7 @@ jobs:
       - name: Generate buildx cache key
         id: generate_buildx_cache_key
         run: |
-          echo "::set-output name=cache_key::$(./container_build/generate_buildx_cache_key.sh)"
+          echo "cache_key=$(./container_build/generate_buildx_cache_key.sh)" >> $GITHUB_OUTPUT
         shell: bash
 
       - name: Cache Hugo Docker layers

build.gradle

@@ -9,9 +9,12 @@ buildscript {
 }
 
 plugins {
-    id "com.github.johnrengelman.shadow" version "8.1.1" apply false
+    //id "com.github.johnrengelman.shadow" version "8.1.1" apply false
+    alias(libs.plugins.shadow) apply false
     //must be applied to all projects including root
-    id "ca.cutterslade.analyze" version "1.9.1" apply true
+    //id "ca.cutterslade.analyze" version "1.9.1" apply true
+    alias(libs.plugins.analyze) apply true
+    id 'java-platform'
 }
 
 def projectGroup = "event-logging-transformer"
@@ -32,19 +35,7 @@ ext.getPropertyOrDefault = { propName, defaultValue ->
 }
 
 ext.versions = [
-    //------event-logging--------------
     eventLogging: getPropertyOrDefault('version', 'SNAPSHOT'),
-
-    //------------3rd-party------------
-    assertj: '3.6.2',
-    javaDiffUtils: '4.0',
-    jackson: '2.8.9',
-    junit: '4.12',
-    logback: '1.2.3',
-    saxon: '9.6.0-6',
-    slf4j: '1.7.22',
-    systemRules: '1.19.0',
-    zzDUMMYzz: 'makes sorting easier'
 ]
 
 allprojects {
@@ -63,21 +54,30 @@ subprojects {
         mavenLocal()
     }
 
+    dependencies {
+        // So deps from the BOM can get the versions
+        implementation platform(libs.junit.bom)
+    }
+
     configurations {
         all {
+            exclude group: "junit", module: "junit"
             exclude group: "org.slf4j", module: "slf4j-log4j12"
             exclude group: "log4j", module: "log4j"
             exclude group: "commons-logging", module: "commons-logging"
 
             resolutionStrategy {
                 dependencySubstitution {
-                    substitute module('log4j:log4j') using module('org.slf4j:log4j-over-slf4j:$versions.slf4j')
+
+                    // Ought to be a way to use the libs.slf4j.log4jOverSlf4j alias
+                    substitute module('log4j:log4j') using module("org.slf4j:log4j-over-slf4j:${libs.versions.slf4j}")
+                    //substitute module('log4j:log4j') using alias(libs.slf4j.log4jOverSlf4j)
                 }
 
                 forcedModules = [
                 ]
             }
         }
     }
-}
 
+}

container_build/docker_pdf/Dockerfile

@@ -1,14 +1,40 @@
-FROM buildkite/puppeteer:10.0.0 
+# See this link for details of how this is put together
+# https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md
+FROM node:20.2.0-buster-slim
+
+# Comment to change cache key
 
 WORKDIR /builder
 
-# Run separately so docker can cache the layer to speed up image re-creation
-RUN apt-get update && apt-get install -y \
-    curl \
-    wait-for-it \
-  && echo "installing npm packages" \
-  && npm install html2canvas jspdf \
-  && mkdir -p /builder/repo
+# Install latest chrome dev package and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others)
+# Note: this installs the necessary libs to make the bundled version of Chromium that Puppeteer
+# installs, work.
+RUN apt-get update \
+    && apt-get install -y wget gnupg \
+    && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
+    && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
+    && echo "installing apt-get packages" \
+    && apt-get update \
+    && apt-get install -y \
+      curl \
+      wait-for-it \
+      google-chrome-stable \
+      fonts-ipafont-gothic \
+      fonts-wqy-zenhei \
+      fonts-thai-tlwg \
+      fonts-kacst \
+      fonts-freefont-ttf \
+      libxss1 \
+      --no-install-recommends \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN npm init -y \
+    && echo "installing npm packages" \
+    && npm install \
+      puppeteer@20.5.0 \
+      html2canvas \
+      jspdf \
+    && mkdir -p /builder/repo
 
 # Pass in the uid/gid of the running user so we can use the same user id
 # in the container so that any files created can be read outside the 
@@ -36,11 +62,24 @@ RUN echo "USER_ID: [$USER_ID]" \
   && if [ ! -n "${user_name}" ]; then adduser --uid "$USER_ID" --system --shell /bin/false --disabled-password --ingroup  "${group_name}" builder; fi \
   && if [ ! -n "${user_name}" ]; then user_name="builder"; fi \
   && echo "user_name: [$user_name]" \
-  && echo "Done"
+  && mkdir -p "/home/${user_name}/Downloads" \
+  && chown -R "${USER_ID}:${GROUP_ID}" "/home/${user_name}" \
+  && chown -R "${USER_ID}:${GROUP_ID}" ./node_modules \
+  && chown -R "${USER_ID}:${GROUP_ID}" ./package.json \
+  && chown -R "${USER_ID}:${GROUP_ID}" ./package-lock.json \
+  && echo "Done setting up users/groups"
 
-ENV  PATH="${PATH}:/node_modules/.bin"
+ENV PATH="${PATH}:/builder/node_modules/.bin"
 
 COPY --chown=$USER_ID:$GROUP_ID generate-pdf.js /builder/
 
 USER $USER_ID
 
+# This needs to be run after we switch to user $USER_ID so it can see the locally
+# installed chromium.
+RUN \
+    cd /builder/node_modules/puppeteer \
+    && pwd \
+    && ls -l \
+    && npm install \
+    && cd /builder

container_build/docker_pdf/generate-pdf.js

@@ -4,11 +4,11 @@ const puppeteer = require('puppeteer');
 var args = process.argv.slice(2);
 var url = args[0]
 
-console.log('Generating pdf for: ', url);
+console.log('Generating pdf for url: ', url);
 
 (async () => {
   const browser = await puppeteer.launch({
-    headless: true,
+    headless: "new",
     // TODO should not be running with no-sandbox really, see
     // https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#setting-up-chrome-linux-sandbox
     args: [
@@ -18,10 +18,19 @@ console.log('Generating pdf for: ', url);
   });
   const page = await browser.newPage();
 
+  await page.setDefaultNavigationTimeout(300000);
   await page.goto(url, {
-    waitUntil: 'networkidle2',
-  });
-  await page.pdf({ path: 'event-logging-schema-docs.pdf', format: 'a4' });
+    waitUntil: 'networkidle2'});
+
+  // If networkidle2 doesn't work then try
+  //await page.goto(url, {
+    //waitUntil: 'domcontentloaded'});
+  //await page.waitForSelector('h1');
+
+  await page.pdf({ 
+    path: 'event-logging-schema-docs.pdf', 
+    format: 'a4', 
+    timeout: 300000 });
 
   await browser.close();
 })();

container_build/runInPupeteerDocker.sh

@@ -52,20 +52,25 @@ docker_login() {
 
 clean_up() {
   echo -e "${GREEN}Stopping schema-hugo-build-env* containers${NC}"
+  # MacOS doesn't support xargs --no-run-if-empty/-r <sigh>, so need a hacky
+  # work around to deal with the case then the ls returns nothing
   docker container \
     ls \
     -q \
     --filter "name=schema-hugo-build-env*" \
-    | xargs -r docker container stop
+    | xargs -I XxX bash -c '[[ -n "XxX" ]] && docker container stop XxX'
 
+  # Seconds
   sleep 1
 
   echo -e "${GREEN}Deleting schema-hugo-build-env* containers${NC}"
+  # MacOS doesn't support xargs --no-run-if-empty/-r <sigh>, so need a hacky
+  # work around to deal with the case then the ls returns nothing
   docker container \
     ls -a \
     -q \
     --filter "name=schema-hugo-build-env*" \
-    | xargs --no-run-if-empty docker container rm
+    | xargs -I XxX bash -c '[[ -n "XxX" ]] && docker container rm XxX'
 
   remove_network
 }
@@ -150,8 +155,8 @@ wait_for_200_response() {
     fi
 
     n=$(( n + 1 ))
-    # sleep for one secs
-    sleep 0.5s
+    # Seconds
+    sleep 0.3
   done
 
   if [ "${were_dots_shown}" = true ]; then
@@ -182,8 +187,7 @@ main() {
 
   if [ "${bash_cmd}" = "bash" ]; then
     run_cmd=( "bash" )
-  elif [ "${bash_cmd}" = "PDF" ]; then
-    #run_cmd=( "node" "../generate-pdf.js" "http://site:1313/all-content/" )
+  elif [[ "${bash_cmd}" = "PDF" || "${bash_cmd}" = "pdf" ]]; then
 
     # Hugo is running in another container so use the service name 'site' as
     # the host
@@ -221,18 +225,32 @@ main() {
   echo -e "${GREEN}Group ID ${BLUE}${group_id}${NC}"
   echo -e "${GREEN}Host repo root dir ${BLUE}${host_abs_repo_dir}${NC}"
 
+  if ! docker version >/dev/null 2>&1; then
+    echo -e "${RED}ERROR: Docker is not installed. Please install Docker or Docker Desktop.${NC}"
+    exit 1
+  fi
+
+  if ! docker buildx version >/dev/null 2>&1; then
+    echo -e "${RED}ERROR: Docker buildx is not installed. Please install it.${NC}"
+    exit 1
+  fi
+
   # So we are not rate limited, login before doing the build as this
   # will pull images
   docker_login
 
-  if ! docker buildx inspect schema-puppeteer-builder >/dev/null 2>&1; then
+  buildx_instance="${image_tag}"
+
+  if ! docker buildx inspect "${buildx_instance}" >/dev/null 2>&1; then
+    echo -e "${GREEN}Creating buildx instance: ${YELLOW}${buildx_instance}${NC}"
     docker buildx \
       create \
-      --name schema-puppeteer-builder
+      --name "${buildx_instance}"
   fi
+
   docker buildx \
     use \
-    schema-puppeteer-builder
+    "${buildx_instance}"
 
   # Make a hash of these things and effectively use this as the cache key for
   # buildx so any change makes it ignore a previous cache.
@@ -262,9 +280,12 @@ main() {
     echo -e "${GREEN}Removing old cache directories${NC}"
     #ls -1trd "${cache_dir_base}/from_"*
 
+    # List all matching dirs
+    # Remove the last item
+    # Delete each item
     ls -1trd "${cache_dir_base}/from_"* \
-      | head -n -1 \
-      | xargs -d '\n' rm -rf --
+      | sed '$d' \
+      | xargs rm -rf --
     echo -e "${GREEN}Remaining cache directories${NC}"
     ls -1trd "${cache_dir_base}/from_"*
   fi
@@ -274,6 +295,7 @@ main() {
   echo -e "${GREEN}Building image ${BLUE}${image_tag}${GREEN}" \
     "(this may take a while on first run)${NC}"
   docker buildx build \
+    --progress=plain \
     --tag "${image_tag}" \
     --build-arg "USER_ID=${user_id}" \
     --build-arg "GROUP_ID=${group_id}" \
@@ -314,13 +336,27 @@ main() {
     --workdir "${dest_dir}" \
     --name "schema_puppeteer-build-env" \
     --network "hugo-schema" \
+    --init \
     --env "BUILD_VERSION=${BUILD_VERSION:-SNAPSHOT}" \
     --env "DOCKER_USERNAME=${DOCKER_USERNAME}" \
     --env "DOCKER_PASSWORD=${DOCKER_PASSWORD}" \
     "${image_tag}" \
     "${run_cmd[@]}"
 
   clean_up
+
+  # Would be nice to use "${bash_cmd,,}" = "pdf" for case insense compare
+  # but that is bash4, and well, you know, macOS :-(
+  if [[ "${bash_cmd}" = "PDF" || "${bash_cmd}" = "pdf" ]]; then
+    pdf_file="${local_repo_root}/event-logging-schema-docs.pdf" 
+    if [[ ! -f "${pdf_file}" ]]; then
+      echo -e "${RED}ERROR${NC} Can't find PDF file ${pdf_file}." \
+        "Has the Puppeteer PDF generation failed?"
+      exit 1
+    fi
+  fi
+
+  echo -e "${GREEN}Done${NC}"
 }
 
 main "$@"

diffAgainstLatestReleases.sh

@@ -142,9 +142,9 @@ main() {
       echo -e "Diffing ${BLUE}${localFile}${NC} against ${BLUE}${downloadedFile}${NC}"
       diffFile="${downloadedFile}.diff"
       #OR with true to stop the exit code from diff stopping the script
-      diff "${localFile}" "${downloadedFile}" > "${diffFile}" || true
+      diff "${downloadedFile}" "${localFile}" > "${diffFile}" || true
       if [ "$( wc -l < "${diffFile}" )" -gt 0 ]; then
-        diff "${localFile}" "${downloadedFile}" || true
+        diff --color "${downloadedFile}" "${localFile}" || true
         echo -e "\n${RED}Warning${NC}: Local schema ${BLUE}${localFile}${NC} differs from the latest release ${BLUE}${downloadedFile}${NC}"
         echo -e "This will likely be as intended but serves as a confirmation of what has changed since the latest release"
         echo -e "(see changes in $PWD/${diffFile})${NC}"

event-logging-transformer-main/build.gradle

@@ -46,20 +46,18 @@ shadowJar {
 // In this section you declare the dependencies for your production and test code
 dependencies {
 
+    implementation libs.bundles.jackson
     // The production code uses the SLF4J logging API at compile time
-    implementation "com.fasterxml.jackson.core:jackson-annotations:$versions.jackson"
-    implementation "com.fasterxml.jackson.core:jackson-databind:$versions.jackson"
-    implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:$versions.jackson"
-    implementation "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:$versions.jackson"
-    implementation "org.slf4j:slf4j-api:${versions.slf4j}"
-
-    runtimeOnly "ch.qos.logback:logback-classic:${versions.logback}"
-    runtimeOnly "net.sf.saxon:Saxon-HE:${versions.saxon}"
-
-    testImplementation "com.github.stefanbirkner:system-rules:${versions.systemRules}"
-    testImplementation "io.github.java-diff-utils:java-diff-utils:${versions.javaDiffUtils}"
-    testImplementation "junit:junit:${versions.junit}"
-    testImplementation "org.assertj:assertj-core:${versions.assertj}"
+    implementation libs.slf4j.api
+
+    runtimeOnly libs.logback.classic
+    runtimeOnly libs.saxon
+
+    testImplementation libs.systemLambda
+    testImplementation libs.javaDiffUtils
+    testImplementation libs.bundles.basicTest.impl
+
+    testRuntimeOnly libs.bundles.basicTest.runtime
 }
 
 run {
@@ -143,6 +141,8 @@ tasks.compileJava.dependsOn validateSchemaVersions
 def failedTestReportFiles = []
 
 test {
+    useJUnitPlatform()
+
     // Use full logging for test exceptions so we can see where the failure occurred 
     testLogging {
         events "failed"
@@ -160,6 +160,13 @@ test {
             failedTestReportFiles.add(pair)
         }
     }
+
+    // Use full logging for test exceptions so we can see where the failure occurred
+    testLogging {
+        events "failed"
+        exceptionFormat = 'full'
+        showStackTraces = true
+    }
 }
 
 gradle.buildFinished {