From 8544232cc9ee5a9c18096d24f1a0a160549746f1 Mon Sep 17 00:00:00 2001 From: Robin de Rooij Date: Tue, 29 Nov 2022 17:22:34 +0100 Subject: [PATCH 1/3] ci: add appimage build again --- .github/workflows/appimage.yml | 49 ++++++++++++++++++++++++ distribution/appimage/AppRun | 40 +++++++++++++++++++ distribution/appimage/README.md | 9 +++++ distribution/appimage/build_script.sh | 15 ++++++++ distribution/appimage/terminusdb.desktop | 7 ++++ distribution/appimage/terminusdb.svg | 1 + 6 files changed, 121 insertions(+) create mode 100644 .github/workflows/appimage.yml create mode 100755 distribution/appimage/AppRun create mode 100644 distribution/appimage/README.md create mode 100755 distribution/appimage/build_script.sh create mode 100644 distribution/appimage/terminusdb.desktop create mode 100644 distribution/appimage/terminusdb.svg diff --git a/.github/workflows/appimage.yml b/.github/workflows/appimage.yml new file mode 100644 index 000000000..2bd0c92b5 --- /dev/null +++ b/.github/workflows/appimage.yml @@ -0,0 +1,49 @@ +name: AppImage distribution + +on: + schedule: + - cron: '45 3 * * *' + workflow_dispatch: + workflow_call: + # TODO: REMOVE THIS BEFORE MERGING + push: + + +jobs: + build: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v3 + + # Runs a set of commands using the runners shell + - name: Run a multi-line script + run: | + sudo apt-add-repository ppa:swi-prolog/stable -y + sudo apt-get update -y + sudo apt-get install swi-prolog-nox libpcre3 libgmp-dev -y + - name: Create appimage + run: | + cd distribution/appimage + wget https://github.com/linuxdeploy/linuxdeploy/releases/download/continuous/linuxdeploy-x86_64.AppImage + chmod +x linuxdeploy-x86_64.AppImage + ./build_script.sh + mv "TerminusDB-$(echo $GITHUB_SHA | cut -c 1-7)-x86_64.AppImage" TerminusDB-amd64.AppImage + + # I want to make sure that we are not secretly relying on SWIPL + # and the tests will still work without SWI Prolog installed + - name: Remove SWI Prolog + run: sudo apt remove swi-prolog-nox + + - name: Run AppImage tests + run: | + cd distribution/appimage && chmod +x TerminusDB-amd64.AppImage && \ + ./TerminusDB-amd64.AppImage store init && ./TerminusDB-amd64.AppImage serve & + cd tests && npm i && export TERMINUSDB_EXEC_PATH=../distribution/appimage/TerminusDB-amd64.AppImage && \ + npm run test + + - name: Archive production artifacts + uses: actions/upload-artifact@v3 + with: + name: TerminusDB-amd64.AppImage + path: distribution/appimage/TerminusDB-amd64.AppImage diff --git a/distribution/appimage/AppRun b/distribution/appimage/AppRun new file mode 100755 index 000000000..80b04f799 --- /dev/null +++ b/distribution/appimage/AppRun @@ -0,0 +1,40 @@ +#!/bin/bash + +function set_cert_paths { + CERT_PATHS="/etc/ssl/certs/ca-certificates.crt +/etc/pki/tls/certs/ca-bundle.crt +/etc/ssl/ca-bundle.pem +/etc/pki/tls/cacert.pem +/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem +/etc/ssl/cert.pem" + + for CERT in $CERT_PATHS + do + if [ -f "$CERT" ]; then + export TERMINUSDB_SYSTEM_SSL_CERTS="$CERT" + fi + + done +} + +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +SOURCE="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" +export SWI_HOME_DIR=$SOURCE/usr/lib/swi-prolog +export TERMINUSDB_DASHBOARD_PATH=${TERMINUSDB_DASHBOARD_PATH:-$SOURCE/usr/share/terminusdb/dashboard} +export TERMINUSDB_SERVER_PACK_DIR=$SOURCE/usr/lib/swi-prolog/pack +export TERMINUSDB_SERVER_DB_PATH=${TERMINUSDB_SERVER_DB_PATH:-$(pwd)/storage} +export TERMINUSDB_SERVER_REGISTRY_PATH=${TERMINUSDB_SERVER_REGISTRY_PATH:-$(pwd)/registry.pl} +export TERMINUSDB_LOG_PATH=${TERMINUSDB_SERVER_LOG_PATH:-$(pwd)/httpd.log} +export TERMINUSDB_SERVER_TMP_PATH=${TERMINUSDB_SERVER_TMP_PATH:-/tmp} +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SOURCE/usr/lib/swi-prolog/lib/x86_64-linux/" + +if [[ -z "${TERMINUSDB_SYSTEM_SSL_CERTS}" ]]; then + set_cert_paths +fi + +$SOURCE/usr/share/terminusdb/terminusdb "$@" diff --git a/distribution/appimage/README.md b/distribution/appimage/README.md new file mode 100644 index 000000000..2993a1eab --- /dev/null +++ b/distribution/appimage/README.md @@ -0,0 +1,9 @@ +# terminusdb appimage + +Run the build\_script.sh on a Ubuntu installation with SWIPL and rust installed. + +And copy the resulting app image in a terminusdb-server folder, run `./TerminusDB-[commit]-x86_64.AppImage` + +# Requirements + +This script is only tested on Ubuntu 16.04 diff --git a/distribution/appimage/build_script.sh b/distribution/appimage/build_script.sh new file mode 100755 index 000000000..38359924f --- /dev/null +++ b/distribution/appimage/build_script.sh @@ -0,0 +1,15 @@ +#!/bin/bash +CURRENT_DIR=$(pwd) +SOURCE="${BASH_SOURCE[0]}" +mkdir -p app_dir/usr/share/terminusdb +mkdir -p app_dir/usr/bin +mkdir -p app_dir/usr/lib/x86_64-linux-gnu +# Build TerminusDB +cd ../../ && make install-deps && make && cd "$CURRENT_DIR" +# Copy relevant files back to appimage dir +cp -r ../../terminusdb ../../dashboard app_dir/usr/share/terminusdb/ +cp -r /usr/lib/swi-prolog app_dir/usr/lib/ +cp -L /usr/lib/x86_64-linux-gnu/libedit.so.2 app_dir/usr/lib/swi-prolog/lib/x86_64-linux/ +cp -L /lib/x86_64-linux-gnu/libpcre.so.3 app_dir/usr/lib/swi-prolog/lib/x86_64-linux/ +cp -L /usr/lib/x86_64-linux-gnu/libbsd.so.0 app_dir/usr/lib/swi-prolog/lib/x86_64-linux/ +LD_LIBRARY_PATH='/usr/lib/swi-prolog/lib/x86_64-linux/' ./linuxdeploy-x86_64.AppImage --appdir ./app_dir --executable /usr/lib/swi-prolog/bin/x86_64-linux/swipl --library /usr/lib/swi-prolog/lib/x86_64-linux/libswipl.so.9 --library /lib/x86_64-linux-gnu/libpcre.so.3 -d terminusdb.desktop -i terminusdb.svg --custom-apprun AppRun --output appimage --verbosity=0 diff --git a/distribution/appimage/terminusdb.desktop b/distribution/appimage/terminusdb.desktop new file mode 100644 index 000000000..5626f0688 --- /dev/null +++ b/distribution/appimage/terminusdb.desktop @@ -0,0 +1,7 @@ +[Desktop Entry] +Categories=Utility; +Type=Application +Icon=terminusdb +Name=TerminusDB +Exec=AppRun +Terminal=true diff --git a/distribution/appimage/terminusdb.svg b/distribution/appimage/terminusdb.svg new file mode 100644 index 000000000..e76b11bb7 --- /dev/null +++ b/distribution/appimage/terminusdb.svg @@ -0,0 +1 @@ + \ No newline at end of file From e463e434fd8cbb81e9227a55ed752b5851fdd817 Mon Sep 17 00:00:00 2001 From: Robin de Rooij Date: Wed, 30 Nov 2022 12:30:26 +0100 Subject: [PATCH 2/3] Enable autoloading Using SWI Prolog without it is broken anyway and it needs to be turned on for the AppImage as I can at least set it up to use a correct SWI_HOME_DIR at runtime. --- src/bootstrap.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap.pl b/src/bootstrap.pl index b4eebf873..998f261ac 100755 --- a/src/bootstrap.pl +++ b/src/bootstrap.pl @@ -41,6 +41,6 @@ foreign(save), undefined(error), toplevel(cli_toplevel), - autoload(false), + autoload(true), stand_alone(true) ]). From f4439214e02fe0ad9a3551f1810e6303ea97d8c0 Mon Sep 17 00:00:00 2001 From: Robin de Rooij Date: Wed, 30 Nov 2022 15:13:42 +0100 Subject: [PATCH 3/3] Set dashboard at runtime Set dashboard path at runtime to allow AppImage dashboard to work --- distribution/appimage/build_script.sh | 2 +- src/config/terminus_config.pl | 4 ++++ src/core/api.pl | 1 + src/core/api/api_init.pl | 14 ++++++++++++++ src/load_paths.pl | 21 ++++++--------------- src/server/main.pl | 1 + 6 files changed, 27 insertions(+), 16 deletions(-) diff --git a/distribution/appimage/build_script.sh b/distribution/appimage/build_script.sh index 38359924f..118acf8ea 100755 --- a/distribution/appimage/build_script.sh +++ b/distribution/appimage/build_script.sh @@ -5,7 +5,7 @@ mkdir -p app_dir/usr/share/terminusdb mkdir -p app_dir/usr/bin mkdir -p app_dir/usr/lib/x86_64-linux-gnu # Build TerminusDB -cd ../../ && make install-deps && make && cd "$CURRENT_DIR" +cd ../../ && make install-dashboard && make install-deps && make && cd "$CURRENT_DIR" # Copy relevant files back to appimage dir cp -r ../../terminusdb ../../dashboard app_dir/usr/share/terminusdb/ cp -r /usr/lib/swi-prolog app_dir/usr/lib/ diff --git a/src/config/terminus_config.pl b/src/config/terminus_config.pl index 3d0caddd3..c7589d8ca 100644 --- a/src/config/terminus_config.pl +++ b/src/config/terminus_config.pl @@ -1,5 +1,6 @@ :- module(config,[ terminusdb_version/1, + dashboard_path/1, bootstrap_config_files/0, server/1, server_name/1, @@ -46,6 +47,9 @@ terminusdb_version('10.1.10'). /* [[[end]]] */ +dashboard_path(Path) :- + getenv('TERMINUSDB_DASHBOARD_PATH', Path). + bootstrap_config_files :- initialize_system_ssl_certs. diff --git a/src/core/api.pl b/src/core/api.pl index ff73da7ab..aa6aca9bf 100644 --- a/src/core/api.pl +++ b/src/core/api.pl @@ -1,6 +1,7 @@ :- module(api, [ % init.pl bootstrap_files/0, + add_dashboard_path/0, index_template/1, initialize_flags/0, initialize_database/2, diff --git a/src/core/api/api_init.pl b/src/core/api/api_init.pl index dd67d9963..8145cfc1c 100644 --- a/src/core/api/api_init.pl +++ b/src/core/api/api_init.pl @@ -1,5 +1,6 @@ :- module(api_init, [ bootstrap_files/0, + add_dashboard_path/0, initialize_flags/0, initialize_database/2, initialize_database_with_store/2, @@ -121,6 +122,19 @@ db_path(DB_Path), initialize_database_with_path(Key, DB_Path, Force). +assert_dashboard_path(Dir) :- + directory_file_path(Dir, 'assets', Assets), + asserta(user:file_search_path(dashboard, Dir)), + asserta(user:file_search_path(assets, Assets)). + +add_dashboard_path :- + config:dashboard_path(Dir), + assert_dashboard_path(Dir). +add_dashboard_path :- + user:file_search_path(terminus_top_dir, Dir), + directory_file_path(Dir, 'dashboard', Dashboard), + assert_dashboard_path(Dashboard). + /* * initialize_database_with_path(Key,DB_Path,Force) is det+error. * diff --git a/src/load_paths.pl b/src/load_paths.pl index 8eca487c8..b47202da2 100644 --- a/src/load_paths.pl +++ b/src/load_paths.pl @@ -11,6 +11,12 @@ prolog_load_context(file, File), relative_file_name(Path, File, '..'). +add_terminus_top_dir :- + top_level_directory(TopDir), + asserta(user:file_search_path(terminus_top_dir, TopDir)). + +:- add_terminus_top_dir. + add_terminus_home_path :- top_level_directory(TopDir), directory_file_path(TopDir, 'src', SrcDir), @@ -122,21 +128,6 @@ directory_file_path(Dir, 'terminusdb-enterprise/test', Enterprise), asserta(user:file_search_path(enterprise_test, Enterprise)). -assert_dashboard_path(Dir) :- - directory_file_path(Dir, 'assets', Assets), - asserta(user:file_search_path(dashboard, Dir)), - asserta(user:file_search_path(assets, Assets)). - -add_dashboard_path :- - getenv('TERMINUSDB_DASHBOARD_PATH', Dir), - assert_dashboard_path(Dir). -add_dashboard_path :- - top_level_directory(Dir), - directory_file_path(Dir, 'dashboard', Dashboard), - assert_dashboard_path(Dashboard). - -:- add_dashboard_path. - :- if(getenv("TERMINUSDB_ENTERPRISE", true)). :- add_enterprise_path. :- add_enterprise_test_path. diff --git a/src/server/main.pl b/src/server/main.pl index 716aa8323..aee5699e9 100644 --- a/src/server/main.pl +++ b/src/server/main.pl @@ -62,6 +62,7 @@ server(Server), server_port(Port), worker_amount(Workers), + add_dashboard_path, load_jwt_conditionally, HTTPOptions = [port(Port), workers(Workers)], foreach(pre_server_startup_hook(Port),true),