diff --git a/.github/workflows/assemble-files.yml b/.github/workflows/assemble-files.yml index 607b078f..53b2658f 100644 --- a/.github/workflows/assemble-files.yml +++ b/.github/workflows/assemble-files.yml @@ -5,30 +5,13 @@ on: jobs: -# cache-files job: -# - Asks cache for rack-box files +# assemble-files job: +# - Downloads certain files for rack-box builds +# - Packages certain files for rack-box builds +# - Uploads rack-box files to artifacts - cache-files: - runs-on: ubuntu-20.04 - outputs: - cache-hit: ${{ steps.cache.outputs.cache-hit }} - - steps: - - name: Ask cache for rack-box files - uses: actions/cache@v3 - id: cache - with: - path: RACK/rack-box/files - key: files-${{ github.sha }} - -# download-files job: -# - Downloads files for rack-box builds -# - Packages files for rack-box builds - - download-files: - needs: cache-files - runs-on: ubuntu-20.04 - if: needs.cache-files.outputs.cache-hit == false + assemble-files: + runs-on: ubuntu-22.04 steps: - name: Start rack-box container (needed for CDR files) @@ -49,17 +32,10 @@ jobs: repository: ge-high-assurance/RACK.wiki path: RACK.wiki - - name: Prepare to cache rack-box files - uses: actions/cache@v3 - id: cache - with: - path: RACK/rack-box/files - key: files-${{ github.sha }} - - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: '3.10' - name: Cache Python dependencies uses: actions/cache@v3 @@ -150,3 +126,10 @@ jobs: shell: bash run: | docker container stop $(docker container ls -qf ancestor=gehighassurance/rack-box:dev) + + - name: Upload rack-box files to artifacts + uses: actions/upload-artifact@v3 + with: + name: rack-box-files + path: RACK/rack-box/files + retention-days: 1 diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml index a1981c1e..71a12f49 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/build-docker-image.yml @@ -15,11 +15,12 @@ on: jobs: # build-docker-image job: +# - Downloads rack-box files from artifacts # - Builds rack-box docker image # - Pushes docker image to Docker Hub build-docker-image: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Check out RACK source @@ -28,18 +29,11 @@ jobs: repository: ge-high-assurance/RACK path: RACK - - name: Ask cache for rack-box files - uses: actions/cache@v3 - id: cache + - name: Download rack-box files from artifacts + uses: actions/download-artifact@v3 with: + name: rack-box-files path: RACK/rack-box/files - key: files-${{ github.sha }} - - - name: We don't have rack-box files? - if: steps.cache.outputs.cache-hit == false - run: | - echo "::error rack-box files are missing" - exit 1 - name: Build rack-box docker image run: | diff --git a/.github/workflows/build-virtual-machine.yml b/.github/workflows/build-virtual-machine.yml index aa84b010..05aac7d0 100644 --- a/.github/workflows/build-virtual-machine.yml +++ b/.github/workflows/build-virtual-machine.yml @@ -11,8 +11,9 @@ on: jobs: # build-virtual-machine job: +# - Downloads rack-box files from artifacts # - Builds rack-box virtual machine -# - Uploads virtual machine to workflow or release +# - Uploads virtual machine to artifacts or releases build-virtual-machine: runs-on: macos-12 @@ -24,24 +25,17 @@ jobs: repository: ge-high-assurance/RACK path: RACK - - name: Ask cache for rack-box files - uses: actions/cache@v3 - id: cache + - name: Download rack-box files from artifacts + uses: actions/download-artifact@v3 with: + name: rack-box-files path: RACK/rack-box/files - key: files-${{ github.sha }} - - - name: We don't have rack-box files? - if: steps.cache.outputs.cache-hit == false - run: | - echo "::error rack-box files are missing" - exit 1 - name: Download base box for virtual machine run: | - curl -LOSfs https://app.vagrantup.com/ubuntu/boxes/focal64/versions/20221115.1.0/providers/virtualbox.box - mkdir -p RACK/rack-box/focal64 - tar -xf virtualbox.box -C RACK/rack-box/focal64 + curl -LOSfs https://app.vagrantup.com/ubuntu/boxes/jammy64/versions/20230215.0.0/providers/virtualbox.box + mkdir -p RACK/rack-box/jammy64 + tar -xf virtualbox.box -C RACK/rack-box/jammy64 rm -f virtualbox.box - name: Build rack-box virtual machine @@ -57,7 +51,7 @@ jobs: split -b 1500m rack-box-${{ inputs.version }}.zip rack-box-${{ inputs.version }}.zip rm rack-box-${{ inputs.version }}.zip - - name: Upload split virtual machine to workflow + - name: Upload virtual machine to artifacts uses: actions/upload-artifact@v3 if: github.event_name != 'release' with: @@ -70,7 +64,7 @@ jobs: # against it; replace it with "gh release upload" if CI fails # run: gh release upload ${{ github.event.release.tag_name }} RACK/rack-box/GitHub-Release-README.md RACK/rack-box/rack-box-${{ inputs.version }}.zip* --clobber - - name: Upload split virtual machine to release + - name: Upload virtual machine to releases uses: softprops/action-gh-release@v1 if: github.event_name == 'release' with: diff --git a/.github/workflows/continuous.yml b/.github/workflows/continuous.yml index ffacb07a..9c0f837b 100644 --- a/.github/workflows/continuous.yml +++ b/.github/workflows/continuous.yml @@ -12,7 +12,7 @@ jobs: # - Lints RACK CLI, RACK Ontology, and shell scripts lint: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Check out RACK source @@ -21,7 +21,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: '3.10' - name: Cache Python dependencies uses: actions/cache@v3 @@ -62,11 +62,13 @@ jobs: uses: ./.github/workflows/assemble-files.yml # test job: +# - Downloads rack-box files from artifacts +# - Installs rack-box on runner # - Runs rack-box tests test: needs: assemble-files - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Check out RACK source @@ -75,16 +77,16 @@ jobs: repository: ge-high-assurance/RACK path: RACK - - name: Ask cache for rack-box files - uses: actions/cache@v3 + - name: Download rack-box files from artifacts + uses: actions/download-artifact@v3 with: + name: rack-box-files path: RACK/rack-box/files - key: files-${{ github.sha }} - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: '3.10' - name: Cache Python dependencies uses: actions/cache@v3 @@ -97,7 +99,7 @@ jobs: ${{ runner.os }}-pip- ${{ runner.os }}- - - name: Install rack-box files on runner + - name: Install rack-box on runner run: | mkdir -p /tmp/files for f in RACK/rack-box/files/*; do diff --git a/.github/workflows/update-wiki.yml b/.github/workflows/update-wiki.yml index 64ce59ed..62d34d13 100644 --- a/.github/workflows/update-wiki.yml +++ b/.github/workflows/update-wiki.yml @@ -13,7 +13,7 @@ jobs: # - Updates RACK-Predefined-Queries.md in RACK.wiki update-wiki: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Check out RACK source @@ -31,14 +31,14 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: '3.10' - name: Cache Python dependencies uses: actions/cache@v3 with: # This path is specific to Ubuntu path: ~/.cache/pip - # Look to see if there is a cache hit + # Look to see if there is a cache hit for cvstomd key: ${{ runner.os }}-pip-csvtomd restore-keys: | ${{ runner.os }}-pip- diff --git a/RACK-Ontology/ontology/PROV-S.sadl b/RACK-Ontology/ontology/PROV-S.sadl index 8270c0ad..061e079f 100644 --- a/RACK-Ontology/ontology/PROV-S.sadl +++ b/RACK-Ontology/ontology/PROV-S.sadl @@ -24,8 +24,7 @@ NODE is a class. THING (note "A piece of data stored in RACK") is a type of NODE. - identifier (note "identifier is any data item that is used to associate items on when loading into the data store.") describes THING with values of type string. - identifier describes THING with at most 1 value. + identifier (note "identifier is any data item that is used to associate items on when loading into the data store.") describes THING with a single value of type string. title (note "A short, human-readable identifying label.") describes THING with values of type string. title describes THING with at most 1 value. description (note "A free-form, multi-line, human-readable explanation of this data element.") describes THING with values of type string. diff --git a/ada/requirements.txt b/ada/requirements.txt index a29bdd71..eed037f4 100644 --- a/ada/requirements.txt +++ b/ada/requirements.txt @@ -1,5 +1,5 @@ colorama~=0.4.4 lxml -rdflib==5.0.0 +rdflib==6.2.0 rope~=0.18.0 git+https://github.com/ge-high-assurance/RACK.git@vr/ScrapingToolKit-master#egg=ScrapingToolKit&subdirectory=ScrapingToolKit diff --git a/cli/README.md b/cli/README.md index a7a28919..b7975ff9 100644 --- a/cli/README.md +++ b/cli/README.md @@ -53,7 +53,7 @@ python3 setup.py install If running on Windows, [GitBash](https://gitforwindows.org/) can be used with the following commands. ```shell -virtualenv venv +python -m venv venv source venv/Scripts/activate pip install --force -r requirements.txt python setup.py install diff --git a/cli/dev/requirements.txt b/cli/dev/requirements.txt index b0c4d3cb..209452b5 100644 --- a/cli/dev/requirements.txt +++ b/cli/dev/requirements.txt @@ -1,8 +1,8 @@ -mypy==0.961 -pylint==2.14.4 -pytest-docker==0.12.0 -pytest==7.1.2 -typed-ast==1.4.3 -types-PyYAML==6.0.10 -types-requests==2.28.1 -types-tabulate==0.8.5 +mypy==1.0.0 +pylint==2.16.2 +pytest-docker==1.0.1 +pytest==7.2.1 +typed-ast==1.5.4 +types-PyYAML==6.0.12.6 +types-requests==2.28.11.13 +types-tabulate==0.9.0.0 diff --git a/cli/optimize.sh b/cli/optimize.sh index 03abcbeb..7dfc9e0f 100755 --- a/cli/optimize.sh +++ b/cli/optimize.sh @@ -4,7 +4,14 @@ set -eu echo "Stopping Fuseki" -systemctl stop fuseki +FUSEKI_PID=$(systemctl show --property MainPID fuseki) +FUSEKI_PID=${FUSEKI_PID#"MainPID="} +if [ -n "${FUSEKI_PID}" ]; then + systemctl stop fuseki + # systemctl doesn't always seem to succeed, so be quite certain + # until we figure out why it doesn't always succeed + kill -9 "${FUSEKI_PID}" > /dev/null 2>&1 +fi RACK_DB="/etc/fuseki/databases/RACK" RACK_FUSEKI_CONFIG="/etc/fuseki/configuration/RACK.ttl" diff --git a/cli/rack/__init__.py b/cli/rack/__init__.py index e7a9efbd..050340e1 100755 --- a/cli/rack/__init__.py +++ b/cli/rack/__init__.py @@ -462,36 +462,36 @@ def manifest( base_path = from_path.parent for step in obj.get('steps',[]): if 'manifest' in step: - path = step['manifest'] + path = Path(step['manifest']) dirname = Path(f'{self.next_fresh():02}_manifest') subdir = to_path.parent.joinpath(dirname) - topath = subdir.joinpath(Path(path).name) + topath = subdir.joinpath(path.name) subdir.mkdir(exist_ok=False) self.manifest(base_path.joinpath(path), topath) - step['manifest'] = str(dirname.joinpath(Path(path).name)) + step['manifest'] = dirname.joinpath(path.name).as_posix() elif 'model' in step: - path = step['model'] + path = Path(step['model']) dirname = Path(f'{self.next_fresh():02}_model') subdir = to_path.parent.joinpath(dirname) - topath = subdir.joinpath(Path(path).name) + topath = subdir.joinpath(path.name) subdir.mkdir(exist_ok=False) self.model(base_path.joinpath(path), topath) - step['model'] = str(dirname.joinpath(Path(path).name)) + step['model'] = dirname.joinpath(path.name).as_posix() elif 'data' in step: - path = step['data'] + path = Path(step['data']) dirname = Path(f'{self.next_fresh():02}_data') subdir = to_path.parent.joinpath(dirname) - topath = subdir.joinpath(Path(path).name) + topath = subdir.joinpath(path.name) subdir.mkdir(exist_ok=False) self.data(base_path.joinpath(path), topath) - step['data'] = str(dirname.joinpath(Path(path).name)) + step['data'] = dirname.joinpath(path.name).as_posix() elif 'nodegroups' in step: - path = step['nodegroups'] + path = Path(step['nodegroups']) dirname = Path(f'{self.next_fresh():02}_nodegroups') subdir = to_path.parent.joinpath(dirname) subdir.mkdir(exist_ok=False) self.nodegroups(base_path.joinpath(path), subdir) - step['nodegroups'] = str(dirname) + step['nodegroups'] = dirname.as_posix() with open(to_path, mode='w', encoding='utf-8-sig', newline='\n') as out: yaml.safe_dump(obj, out) diff --git a/cli/requirements.txt b/cli/requirements.txt index 084a7343..4bd4194e 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -1,19 +1,19 @@ ase==3.22.1 -attrs==21.4.0 +attrs==22.2.0 certifi==2022.12.7 -chardet==5.0.0 -colorama==0.4.5 -idna==3.3 -importlib-metadata==4.12.0 -jsonschema==3.2.0 -pyrsistent==0.18.1 +chardet==5.1.0 +colorama==0.4.6 +idna==3.4 +importlib-metadata==6.0.0 +jsonschema==4.17.3 +pyrsistent==0.19.3 python-dateutil==2.8.2 -PyYAML==5.4.1 -requests==2.28.1 -Pillow==9.0.1 -plotly==5.9.0 -semtk-python3 @ git+https://github.com/ge-semtk/semtk-python3@dadfade1537ea77c30a777cb544e55b2fc7e4790 +PyYAML==6.0 +requests==2.28.2 +Pillow==9.4.0 +plotly==5.13.0 +semtk-python3 @ git+https://github.com/ge-semtk/semtk-python3@3794a10ba5c2065b145d88f074a7e52028c21cdb six==1.16.0 -tabulate==0.8.10 -urllib3==1.26.10 -zipp==3.8.1 +tabulate==0.9.0 +urllib3==1.26.14 +zipp==3.13.0 diff --git a/cli/restart-fuseki.sh b/cli/restart-fuseki.sh new file mode 100755 index 00000000..1067e2a0 --- /dev/null +++ b/cli/restart-fuseki.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Copyright (c) 2023, General Electric Company and Galois, Inc. + +set -e + +FUSEKI_PID=$(systemctl show --property MainPID fuseki) +FUSEKI_PID=${FUSEKI_PID#"MainPID="} + +if [ -n "${FUSEKI_PID}" ]; then + systemctl stop fuseki + # systemctl doesn't always seem to succeed, so be quite certain + # until we figure out why it doesn't always succeed + kill -9 "${FUSEKI_PID}" > /dev/null 2>&1 +fi + +systemctl start fuseki diff --git a/rack-box/README.md b/rack-box/README.md index 7ef8fc0d..86e1f3ea 100644 --- a/rack-box/README.md +++ b/rack-box/README.md @@ -76,9 +76,9 @@ although we will mention each file here as well: --exclude=.github --exclude=assist --exclude=cli --exclude=rack-box --exclude=tests --exclude=tools RACK`) -- `focal64\*`: Unpack a recent Ubuntu vagrant box here (`curl -LOSfs - https://app.vagrantup.com/ubuntu/boxes/focal64/versions/20221021.0.0/providers/virtualbox.box - && tar -xf virtualbox.box -C RACK/rack-box/focal64`) +- `jammy64\*`: Unpack a recent Ubuntu vagrant box here (`curl -LOSfs + https://app.vagrantup.com/ubuntu/boxes/jammy64/versions/20230215.0.0/providers/virtualbox.box + && tar -xf virtualbox.box -C RACK/rack-box/jammy64`) Once you have put these files into the `files` subdirectory, skip to [Build the rack-box images](#Build-the-rack-box-images) for the next diff --git a/rack-box/rack-box-docker.json b/rack-box/rack-box-docker.json index 0d668f01..94208a8b 100644 --- a/rack-box/rack-box-docker.json +++ b/rack-box/rack-box-docker.json @@ -16,7 +16,7 @@ "ENTRYPOINT [ \"/usr/bin/python3\", \"/usr/bin/systemctl\" ]" ], "commit": true, - "image": "ubuntu:20.04", + "image": "ubuntu:22.04", "pull": false } ], diff --git a/rack-box/rack-box-hyperv.json b/rack-box/rack-box-hyperv.json index bca8d82d..ca67eb2a 100644 --- a/rack-box/rack-box-hyperv.json +++ b/rack-box/rack-box-hyperv.json @@ -15,19 +15,7 @@ { "type": "hyperv-iso", - "boot_command": [ - "", - "", - "", - "", - "", - "", - "", - "", - "", - "autoinstall ds=nocloud-net;s=http://{{.HTTPIP}}:{{.HTTPPort}}/", - "" - ], + "boot_command": ["cset gfxpayload=keeplinux /casper/vmlinuz quiet autoinstall ds=nocloud-net\\;s=http://{{.HTTPIP}}:{{.HTTPPort}}/ubuntu/ ---initrd /casper/initrdboot"], "boot_wait": "2s", "communicator": "ssh", "cpus": "{{user `cpus`}}", @@ -35,8 +23,8 @@ "enable_dynamic_memory": true, "headless": "{{user `headless`}}", "http_directory": "http", - "iso_checksum": "sha256:5035be37a7e9abbdc09f0d257f3e33416c1a0fb322ba860d42d74aa75c3468d4", - "iso_url": "https://releases.ubuntu.com/focal/ubuntu-20.04.5-live-server-amd64.iso", + "iso_checksum": "sha256:10f19c5b2b8d6db711582e0e27f5116296c34fe4b313ba45f9b201a5007056cb", + "iso_url": "http://releases.ubuntu.com/jammy/ubuntu-22.04.1-live-server-amd64.iso", "memory": "{{user `memory`}}", "shutdown_command": "sudo shutdown -P now", "ssh_password": "ubuntu", diff --git a/rack-box/rack-box-virtualbox.json b/rack-box/rack-box-virtualbox.json index f8bbd1ed..092f150c 100644 --- a/rack-box/rack-box-virtualbox.json +++ b/rack-box/rack-box-virtualbox.json @@ -20,7 +20,7 @@ "headless": "{{user `headless`}}", "http_directory": "http", "shutdown_command": "sudo shutdown -P now", - "source_path": "focal64/box.ovf", + "source_path": "jammy64/box.ovf", "ssh_private_key_file": "vagrant", "ssh_username": "vagrant", "vboxmanage": [ diff --git a/rack-ui/pages/utility.py b/rack-ui/pages/utility.py index e5ac5df2..2c48e7e4 100644 --- a/rack-ui/pages/utility.py +++ b/rack-ui/pages/utility.py @@ -49,7 +49,8 @@ def run_restart(n_clicks): raise Exception("Triple store restart not supported in this deployment") # restart fuseki - completed_process = run_subprocess("sudo systemctl restart fuseki", get_temp_dir_unique("restart-fuseki")) + command = "sudo ../cli/restart-fuseki.sh" + completed_process = run_subprocess(command, get_temp_dir_unique("restart-fuseki")) if completed_process.returncode == 0: return dcc.Markdown("Restarted the triple store.") else: diff --git a/rack-ui/requirements.txt b/rack-ui/requirements.txt index e24ba6dc..de4e1faa 100644 --- a/rack-ui/requirements.txt +++ b/rack-ui/requirements.txt @@ -1,2 +1,2 @@ dash[diskcache] -dash-bootstrap-components[pandas] \ No newline at end of file +dash-bootstrap-components[pandas] diff --git a/tests/requirements.txt b/tests/requirements.txt index 4f86983c..965bee53 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,3 +1,3 @@ -pytest==6.1.2 -pytest-docker==0.10.1 -requests==2.25.0 +pytest==7.2.1 +pytest-docker==1.0.1 +requests==2.28.2