Merge branch 'master' into TESTING_updates_kwq

.github/workflows/assemble-files.yml

@@ -5,30 +5,13 @@ on:
 
 jobs:
 
-# cache-files job:
-#  - Asks cache for rack-box files
+# assemble-files job:
+#  - Downloads certain files for rack-box builds
+#  - Packages certain files for rack-box builds
+#  - Uploads rack-box files to artifacts
 
-  cache-files:
-    runs-on: ubuntu-20.04
-    outputs:
-      cache-hit: ${{ steps.cache.outputs.cache-hit }}
-
-    steps:
-    - name: Ask cache for rack-box files
-      uses: actions/cache@v3
-      id: cache
-      with:
-        path: RACK/rack-box/files
-        key: files-${{ github.sha }}
-
-# download-files job:
-#  - Downloads files for rack-box builds
-#  - Packages files for rack-box builds
-
-  download-files:
-    needs: cache-files
-    runs-on: ubuntu-20.04
-    if: needs.cache-files.outputs.cache-hit == false
+  assemble-files:
+    runs-on: ubuntu-22.04
 
     steps:
     - name: Start rack-box container (needed for CDR files)
@@ -49,17 +32,10 @@ jobs:
         repository: ge-high-assurance/RACK.wiki
         path: RACK.wiki
 
-    - name: Prepare to cache rack-box files
-      uses: actions/cache@v3
-      id: cache
-      with:
-        path: RACK/rack-box/files
-        key: files-${{ github.sha }}
-
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: 3.8
+        python-version: '3.10'
 
     - name: Cache Python dependencies
       uses: actions/cache@v3
@@ -75,17 +51,17 @@ jobs:
     - name: Download Fuseki
       shell: bash
       run: |
-        curl -LSfs https://archive.apache.org/dist/jena/binaries/apache-jena-fuseki-4.6.1.tar.gz -o RACK/rack-box/files/fuseki.tar.gz
+        curl -LSfs https://archive.apache.org/dist/jena/binaries/apache-jena-fuseki-4.7.0.tar.gz -o RACK/rack-box/files/fuseki.tar.gz
 
     - name: Download Apache Jena
       shell: bash
       run: |
-        curl -LSfs https://archive.apache.org/dist/jena/binaries/apache-jena-4.6.1.tar.gz -o RACK/rack-box/files/jena.tar.gz
+        curl -LSfs https://archive.apache.org/dist/jena/binaries/apache-jena-4.7.0.tar.gz -o RACK/rack-box/files/jena.tar.gz
 
     - name: Download SemTK
       shell: bash
       run: |
-        curl -LSfs https://github.com/ge-semtk/semtk/releases/download/v2.5.0-20230110/semtk-opensource-v2.5.0-20230110-dist.tar.gz -o RACK/rack-box/files/semtk.tar.gz
+        curl -LSfs https://github.com/ge-semtk/semtk/releases/download/v2.5.0-20230323/semtk-opensource-v2.5.0-20230323-dist.tar.gz -o RACK/rack-box/files/semtk.tar.gz
 
     - name: Download CSS stylesheet
       shell: bash
@@ -95,7 +71,7 @@ jobs:
     - name: Download systemctl script
       shell: bash
       run: |
-        curl -LSfs https://github.com/gdraheim/docker-systemctl-replacement/raw/v1.5.4505/files/docker/systemctl3.py -o RACK/rack-box/files/systemctl3.py
+        curl -LSfs https://github.com/gdraheim/docker-systemctl-replacement/raw/v1.5.7106/files/docker/systemctl3.py -o RACK/rack-box/files/systemctl3.py
 
     - name: Build OWL files using sadl-eclipse
       shell: bash
@@ -150,3 +126,10 @@ jobs:
       shell: bash
       run: |
         docker container stop $(docker container ls -qf ancestor=gehighassurance/rack-box:dev)
+
+    - name: Upload rack-box files to artifacts
+      uses: actions/upload-artifact@v3
+      with:
+        name: rack-box-files
+        path: RACK/rack-box/files
+        retention-days: 1

.github/workflows/build-docker-image.yml

@@ -15,11 +15,12 @@ on:
 jobs:
 
 # build-docker-image job:
+#  - Downloads rack-box files from artifacts
 #  - Builds rack-box docker image
 #  - Pushes docker image to Docker Hub
 
   build-docker-image:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     steps:
     - name: Check out RACK source
@@ -28,18 +29,11 @@ jobs:
         repository: ge-high-assurance/RACK
         path: RACK
 
-    - name: Ask cache for rack-box files
-      uses: actions/cache@v3
-      id: cache
+    - name: Download rack-box files from artifacts
+      uses: actions/download-artifact@v3
       with:
+        name: rack-box-files
         path: RACK/rack-box/files
-        key: files-${{ github.sha }}
-
-    - name: We don't have rack-box files?
-      if: steps.cache.outputs.cache-hit == false
-      run: |
-        echo "::error rack-box files are missing"
-        exit 1
 
     - name: Build rack-box docker image
       run: |

.github/workflows/build-virtual-machine.yml

@@ -11,8 +11,9 @@ on:
 jobs:
 
 # build-virtual-machine job:
+#  - Downloads rack-box files from artifacts
 #  - Builds rack-box virtual machine
-#  - Uploads virtual machine to workflow or release
+#  - Uploads virtual machine to artifacts or releases
 
   build-virtual-machine:
     runs-on: macos-12
@@ -24,24 +25,17 @@ jobs:
         repository: ge-high-assurance/RACK
         path: RACK
 
-    - name: Ask cache for rack-box files
-      uses: actions/cache@v3
-      id: cache
+    - name: Download rack-box files from artifacts
+      uses: actions/download-artifact@v3
       with:
+        name: rack-box-files
         path: RACK/rack-box/files
-        key: files-${{ github.sha }}
-
-    - name: We don't have rack-box files?
-      if: steps.cache.outputs.cache-hit == false
-      run: |
-        echo "::error rack-box files are missing"
-        exit 1
 
     - name: Download base box for virtual machine
       run: |
-        curl -LOSfs https://app.vagrantup.com/ubuntu/boxes/focal64/versions/20221115.1.0/providers/virtualbox.box
-        mkdir -p RACK/rack-box/focal64
-        tar -xf virtualbox.box -C RACK/rack-box/focal64
+        curl -LOSfs https://app.vagrantup.com/ubuntu/boxes/jammy64/versions/20230302.0.0/providers/virtualbox.box
+        mkdir -p RACK/rack-box/jammy64
+        tar -xf virtualbox.box -C RACK/rack-box/jammy64
         rm -f virtualbox.box
 
     - name: Build rack-box virtual machine
@@ -57,7 +51,7 @@ jobs:
         split -b 1500m rack-box-${{ inputs.version }}.zip rack-box-${{ inputs.version }}.zip
         rm rack-box-${{ inputs.version }}.zip
 
-    - name: Upload split virtual machine to workflow
+    - name: Upload virtual machine to artifacts
       uses: actions/upload-artifact@v3
       if: github.event_name != 'release'
       with:
@@ -70,7 +64,7 @@ jobs:
     # against it; replace it with "gh release upload" if CI fails
     # run: gh release upload ${{ github.event.release.tag_name }} RACK/rack-box/GitHub-Release-README.md RACK/rack-box/rack-box-${{ inputs.version }}.zip* --clobber
 
-    - name: Upload split virtual machine to release
+    - name: Upload virtual machine to releases
       uses: softprops/action-gh-release@v1
       if: github.event_name == 'release'
       with:

.github/workflows/continuous.yml

@@ -12,7 +12,7 @@ jobs:
 #  - Lints RACK CLI, RACK Ontology, and shell scripts
 
   lint:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     steps:
     - name: Check out RACK source
@@ -21,7 +21,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: 3.8
+        python-version: '3.10'
 
     - name: Cache Python dependencies
       uses: actions/cache@v3
@@ -62,11 +62,13 @@ jobs:
     uses: ./.github/workflows/assemble-files.yml
 
 # test job:
+#  - Downloads rack-box files from artifacts
+#  - Installs rack-box on runner
 #  - Runs rack-box tests
 
   test:
     needs: assemble-files
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     steps:
     - name: Check out RACK source
@@ -75,16 +77,16 @@ jobs:
         repository: ge-high-assurance/RACK
         path: RACK
 
-    - name: Ask cache for rack-box files
-      uses: actions/cache@v3
+    - name: Download rack-box files from artifacts
+      uses: actions/download-artifact@v3
       with:
+        name: rack-box-files
         path: RACK/rack-box/files
-        key: files-${{ github.sha }}
 
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: 3.8
+        python-version: '3.10'
 
     - name: Cache Python dependencies
       uses: actions/cache@v3
@@ -97,7 +99,7 @@ jobs:
           ${{ runner.os }}-pip-
           ${{ runner.os }}-
 
-    - name: Install rack-box files on runner
+    - name: Install rack-box on runner
       run: |
         mkdir -p /tmp/files
         for f in RACK/rack-box/files/*; do

.github/workflows/update-wiki.yml

@@ -13,7 +13,7 @@ jobs:
 #  - Updates RACK-Predefined-Queries.md in RACK.wiki
 
   update-wiki:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     steps:
     - name: Check out RACK source
@@ -31,14 +31,14 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: 3.8
+        python-version: '3.10'
 
     - name: Cache Python dependencies
       uses: actions/cache@v3
       with:
         # This path is specific to Ubuntu
         path: ~/.cache/pip
-        # Look to see if there is a cache hit
+        # Look to see if there is a cache hit for cvstomd
         key: ${{ runner.os }}-pip-csvtomd
         restore-keys: |
           ${{ runner.os }}-pip-

.gitignore

@@ -70,3 +70,5 @@ rack-ui/cache/
 rack-ui/.project
 EntityResolution/.project
 EntityResolution/Resolutions/Summary.csv
+EntityResolution/cache/
+EntityResolution/Resolutions/*.json

RACK-Ontology/ontology/PROV-S.sadl

@@ -24,8 +24,7 @@ NODE is a class.
 
 
 THING (note "A piece of data stored in RACK") is a type of NODE.
-	identifier (note "identifier is any data item that is used to associate items on when loading into the data store.") describes THING with values of type string.
-	identifier describes THING with at most 1 value.
+	identifier (note "identifier is any data item that is used to associate items on when loading into the data store.") describes THING with a single value of type string.
 	title (note "A short, human-readable identifying label.") describes THING with values of type string.
 	title describes THING with at most 1 value.
 	description (note "A free-form, multi-line, human-readable explanation of this data element.") describes THING with values of type string.

ada/requirements.txt

@@ -1,5 +1,5 @@
 colorama~=0.4.4
 lxml
-rdflib==5.0.0
+rdflib==6.2.0
 rope~=0.18.0
 git+https://github.com/ge-high-assurance/RACK.git@vr/ScrapingToolKit-master#egg=ScrapingToolKit&subdirectory=ScrapingToolKit

cli/README.md

@@ -53,7 +53,7 @@ python3 setup.py install
 If running on Windows, [GitBash](https://gitforwindows.org/) can be used with the following commands.
 
 ```shell
-virtualenv venv
+python -m venv venv
 source venv/Scripts/activate
 pip install --force -r requirements.txt
 python setup.py install

cli/dev/requirements.txt

@@ -1,8 +1,8 @@
-mypy==0.961
-pylint==2.14.4
-pytest-docker==0.12.0
-pytest==7.1.2
-typed-ast==1.4.3
-types-PyYAML==6.0.10
-types-requests==2.28.1
-types-tabulate==0.8.5
+mypy==1.0.0
+pylint==2.16.2
+pytest-docker==1.0.1
+pytest==7.2.1
+typed-ast==1.5.4
+types-PyYAML==6.0.12.6
+types-requests==2.28.11.13
+types-tabulate==0.9.0.0

cli/optimize.sh

@@ -3,13 +3,26 @@
 
 set -eu
 
-echo "Stopping Fuseki"
-systemctl stop fuseki
+if systemctl is-active --quiet fuseki; then
+
+    echo "Stopping Fuseki"
+
+    # remember the PID in case stopping fuseki doesn't stop it    
+    FUSEKI_PID=$(systemctl show --property MainPID fuseki)
+    FUSEKI_PID=${FUSEKI_PID#"MainPID="}
+
+    systemctl stop fuseki
+
+    # systemctl stop doesn't always work, so kill -9 for now to be sure
+    if [ -n "${FUSEKI_PID}" ]; then
+        kill -9 "${FUSEKI_PID}" &> /dev/null || true
+    fi
+fi
 
 RACK_DB="/etc/fuseki/databases/RACK"
 RACK_FUSEKI_CONFIG="/etc/fuseki/configuration/RACK.ttl"
-# Currently using TDB1, this should become tdb2.tdbstats for TDB2
-TDBSTATS="/opt/jena/bin/tdbstats"
+# Currently using TDB2, revert to tdbstats for TDB1
+TDBSTATS="/opt/jena/bin/tdb2.tdbstats"
 TMP_STATS="/tmp/stats.opt"
 # Usually it goes into a `Data-0001` subdirectory, but it seems that with our
 # RACK.ttl configuration we just don't have the extra directory.