Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
aucampia committed Nov 23, 2021
1 parent 7c7c129 commit 42a24cf
Show file tree
Hide file tree
Showing 4 changed files with 224 additions and 9 deletions.
4 changes: 4 additions & 0 deletions .editorconfig
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ trim_trailing_whitespace = false
[*.{js, py, pyi, toml, yml, yaml}]
charset = utf-8

[*.{yaml, yml}]
indent_style = space
indent_size = 2

# 4 space indentation
[*.py]
indent_style = space
Expand Down
35 changes: 26 additions & 9 deletions .github/workflows/validate.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,15 @@ on: [push, pull_request, workflow_dispatch]

env:
FORCE_COLOR: 1
XDG_CACHE_HOME: ${{ github.workspace }}/cache

jobs:
validate:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
python-version: ["3.7"]
# python-version: ["3.7", "3.8", "3.9"]
python-version: ["3.7", "3.8", "3.9"]
# ubuntu-latest, macos-latest,
# os: [ windows-latest]
os: [ubuntu-latest, macos-latest]
Expand Down Expand Up @@ -42,39 +42,56 @@ jobs:
with:
python-version: ${{ matrix.python-version }}

- uses: actions/setup-java@v2
with:
distribution: "temurin"
java-version: "17"

- name: Get pip cache dir
id: pip-cache
shell: bash
run: |
python -m ensurepip --upgrade
echo "::set-output name=dir::$(pip cache dir)"
- name: Cache
- name: Cache pip
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ matrix.os }}-pip-${{ matrix.python-version }}-v1-${{
hashFiles('**/setup.py', '**/requirements*.txt') }}
restore-keys: |
${{ matrix.os }}-pip-${{ matrix.python-version }}-v1-
- name: Cache xdg
uses: actions/cache@v2
with:
path: ${{ env.XDG_CACHE_HOME }}
key: ${{ matrix.os }}-xdg-v1-${{ hashFiles('**/with-fuseki.sh') }}
restore-keys: |
${{ matrix.os }}-xdg-v1-
- name: Install dependencies
shell: bash
run: |
if [ "${{ matrix.os-type }}" == "linux" ]
then
bash .travis.fuseki_install_optional.sh
fi
# bash .travis.fuseki_install_optional.sh
pip install --default-timeout 60 -r requirements.txt
pip install --default-timeout 60 -r requirements.dev.txt
pip install networkx
if [ "${{ matrix.os }}" == "ubuntu-latest" ]
then
sudo apt-get install -y libdb-dev
pip install --user --upgrade berkeleydb
elif [ "${{ matrix.os }}" == "macos-latest" ]
then
brew install berkeley-db@4
export BERKELEYDB_DIR=$(brew --prefix berkeley-db@4)
pip install --user --upgrade berkeleydb
fi
python setup.py install
- name: Validate
shell: bash
run: |
black --config black.toml --check ./rdflib || true
flake8 --exit-zero rdflib
pytest --cov test/testutils.py test/test_graph_http.py
pytest --cov
./with-fuseki.sh pytest -ra --cov
# - name: Upload coverage
# uses: codecov/codecov-action@v2
# with:
Expand Down
26 changes: 26 additions & 0 deletions docker-compose.tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,29 @@ services:
- .:/rdflib
working_dir: /rdflib
command: ["python", "-m", "mypy", "--show-error-context", "--show-error-codes" ,"rdflib"]

fuseki-server:
image: docker.io/stain/jena-fuseki:3.14.0
restart: always
environment: {}
ports:
- 3030:3030/tcp
entrypoint: []
command:
- bash
- -c
- |
exec /jena-fuseki/fuseki-server --port 3030 --debug --update --mem /db
fuseki-server-tdb:
image: docker.io/stain/jena-fuseki:3.14.0
restart: always
environment: {}
ports:
- 3031:3031/tcp
entrypoint: []
command:
- bash
- -c
- |
exec /jena-fuseki/fuseki-server --port 3031 --debug --update --memTDB --set tdb:unionDefaultGraph=true /db
168 changes: 168 additions & 0 deletions with-fuseki.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
#!/usr/bin/env bash

# script_name="${0}"
script_dirname="$( dirname -- "${0}" )"
# script_basename="$( basename -- "${0}" )"

set -eo pipefail

wait_http_okay() {
local timeout=10
for var in "${@}"
do
eval "local ${var}"
done
1>&2 echo "INFO: checking if ${url} (${label}) is okay"
for ((i=1;i<=${timeout};i++))
do
curl --silent "${url}" --fail -o /dev/null && {
1>&2 echo "INFO: url ${url} (${label}) is okay"
return 0
}
kill -0 "${pid}" || {
1>&2 echo "ERROR: pid ${pid} (${label}) not running anymore ..."
return 1
}
1>&2 echo "INFO: url ${url} (${label}) not okay, waiting ..."
sleep 2
done
1>&2 echo "ERROR: failed to load ${url} (${label})"
return 1
}

assert_pid_running() {
for var in "${@}"
do
eval "local ${var}"
done

kill -0 "${pid}" || {
1>&2 echo "ERROR: pid ${pid} (${label}) not running anymore ..."
return 1
}
1>&2 echo "INFO: pid ${pid} (${label}) running"
return 0
}

kill_and_wait_pid() {
local timeout=10
for var in "${@}"
do
eval "local ${var}"
done
1>&2 echo "INFO: killing pid ${pid} (${label})"
kill "${pid}"
for ((i=1;i<=${timeout};i++))
do
kill -0 "${pid}" 2>/dev/null 1>&2 || {
1>&2 echo "INFO: pid ${pid} (${label}) is dead "
return 0
}
1>&2 echo "INFO: pid ${pid} (${label}) is not dead, waiting ..."
sleep 2
done
1>&2 echo "ERROR: wait timeout for pid ${pid} (${label})"
return 1
}

fuseki_pid_normal=""
fuseki_pid_tdb=""

exit_handler() {
1>&2 declare -p fuseki_pid_normal fuseki_pid_tdb
if [ -n "${fuseki_pid_normal}" ]
then
kill_and_wait_pid "pid=${fuseki_pid_normal}" "label=fuseki-normal" || :
fi
if [ -n "${fuseki_pid_tdb}" ]
then
kill_and_wait_pid "pid=${fuseki_pid_tdb}" "label=fuseki-tdb" || :
fi

if [ -n "${FUSEKI_DUMP_LOGS}" ] || [ "${xrc}" != "0" ]
then
1>&2 echo "dumping ${fuseki_base_tdb}/out.log"
cat "${fuseki_base_tdb}/out.log" || :
1>&2 echo "dumping ${fuseki_base_normal}/out.log"
cat "${fuseki_base_normal}/out.log" || :
fi
}


main() {
: "${LOCALSTATEDIR:=${script_dirname}/var}"
# : ${FUSEKI_PORT:=3030}
: "${XDG_CACHE_HOME:=${HOME}/.cache}"

local jena_uri="https://archive.apache.org/dist/jena/binaries/apache-jena-fuseki-3.17.0.tar.gz"
local jena_sha512="2b92f3304743da335f648c1be7b5d7c3a94725ed0a9b5123362c89a50986690114dcef0813e328126b14240f321f740b608cc353417e485c9235476f059bd380"
local jena_archive_basename
jena_archive_basename="$(basename "${jena_uri}")"
local jena_archive="${XDG_CACHE_HOME}/${jena_archive_basename}"
local jena_stem="${jena_archive_basename%%.tar.gz}"

1>&2 declare -p jena_uri jena_archive XDG_CACHE_HOME LOCALSTATEDIR
if ! [ -e "${jena_archive}" ]
then
mkdir -vp "${XDG_CACHE_HOME}"
curl "${jena_uri}" -o "${jena_archive}"
fi
printf "%s %s\n" "${jena_sha512}" "${jena_archive_basename}" > "${jena_archive}.sha512"
(cd "${XDG_CACHE_HOME}" && shasum -c -a512 "${jena_archive_basename}.sha512") || {
echo 1>&2 "ERROR: digest verification failed"
rm -rv "${jena_archive}" "${jena_archive}.sha512"
}
mkdir -vp "${LOCALSTATEDIR}"
tar -zxf "${jena_archive}" -C "${LOCALSTATEDIR}"

local FUSEKI_HOME=${LOCALSTATEDIR}/${jena_stem}
declare -p FUSEKI_HOME
export FUSEKI_HOME

fuseki_base_normal="${FUSEKI_HOME}/run-normal"
fuseki_base_tdb="${FUSEKI_HOME}/run-tdb"
local fuseki_pidfile_normal="${fuseki_base_normal}/server.pid"
local fuseki_pidfile_tdb="${fuseki_base_tdb}/server.pid"

mkdir -vp "${fuseki_base_normal}" "${fuseki_base_tdb}"
local fuseki_port_normal=3030
local fuseki_port_tdb=3031

1>&2 declare -p fuseki_base_normal fuseki_base_tdb fuseki_pidfile_normal fuseki_pidfile_tdb fuseki_port_normal fuseki_port_tdb

trap exit_handler EXIT

1>&2 echo "INFO: starting fuseki: normal"
FUSEKI_BASE="${fuseki_base_normal}" bash "${FUSEKI_HOME}/fuseki-server" \
--port "${fuseki_port_normal}" --debug \
--update --mem /db &>"${fuseki_base_normal}/out.log" &
fuseki_pid_normal="${!}"
echo "${fuseki_pid_normal}" > "${fuseki_pidfile_normal}"

1>&2 echo "INFO: starting fuseki: tdb"
FUSEKI_BASE="${fuseki_base_tdb}" bash "${FUSEKI_HOME}/fuseki-server" \
--port "${fuseki_port_tdb}" --debug \
--update --memTDB --set tdb:unionDefaultGraph=true /db &>"${fuseki_base_tdb}/out.log" &
fuseki_pid_tdb="${!}"
echo "${fuseki_pid_tdb}" > "${fuseki_pidfile_tdb}"

declare -p fuseki_pid_tdb fuseki_pid_normal

wait_http_okay "pid=${fuseki_pid_normal}" "url=http://localhost:${fuseki_port_normal}/" "label=fuseki-normal"
wait_http_okay "pid=${fuseki_pid_tdb}" "url=http://localhost:${fuseki_port_tdb}/" "label=fuseki-tdb"

assert_pid_running "pid=${fuseki_pid_tdb}" "label=fuseki-tdb"
assert_pid_running "pid=${fuseki_pid_normal}" "label=fuseki-normal"

local -a args=("${@}")

1>&2 echo "running: ${args[*]}"

xrc=0
"${args[@]}" || {
xrc="${?}"
}
exit "${xrc}"
}

main "${@}"

0 comments on commit 42a24cf

Please sign in to comment.