Add GitHub actions workflow for validation

.drone.yml

@@ -21,15 +21,15 @@ steps:
   - export COVERALLS_GIT_REPO="$DRONE_REPO_NAME"
   - export COVERALLS_GIT_BRANCH="$DRONE_SOURCE_BRANCH"
   - export CI_BRANCH="$DRONE_SOURCE_BRANCH"
-  - bash .travis.fuseki_install_optional.sh
+  - apt-get update && apt-get install -y openjdk-11-jdk-headless
   - pip install --default-timeout 60 -r requirements.txt
   - pip install --default-timeout 60 -r requirements.dev.txt
   - pip install --default-timeout 60 coveralls && export HAS_COVERALLS=1
   - python setup.py install
   - black --config black.toml --check ./rdflib || true
   - flake8 --exit-zero rdflib
   - mypy --show-error-context --show-error-codes rdflib
-  - pytest --cov
+  - ./with-fuseki.sh pytest -ra --cov
   - coveralls
 
 ---
@@ -44,13 +44,13 @@ steps:
 - name: test
   image: python:3.8
   commands:
-  - bash .travis.fuseki_install_optional.sh
+  - apt-get update && apt-get install -y openjdk-11-jdk-headless
   - pip install --default-timeout 60 -r requirements.txt
   - pip install --default-timeout 60 -r requirements.dev.txt
   - python setup.py install
   - black --config black.toml --check ./rdflib || true
   - flake8 --exit-zero rdflib
-  - pytest
+  - ./with-fuseki.sh pytest -ra
 
 ---
 kind: pipeline
@@ -64,10 +64,10 @@ steps:
 - name: test
   image: python:3.9
   commands:
-  - bash .travis.fuseki_install_optional.sh
+  - apt-get update && apt-get install -y openjdk-11-jdk-headless
   - pip install --default-timeout 60 -r requirements.txt
   - pip install --default-timeout 60 -r requirements.dev.txt
   - python setup.py install
   - black --config black.toml --check ./rdflib || true
   - flake8 --exit-zero rdflib
-  - pytest
+  - ./with-fuseki.sh pytest -ra

.editorconfig

@@ -20,6 +20,10 @@ trim_trailing_whitespace = false
 [*.{js, py, pyi, toml, yml, yaml}]
 charset = utf-8
 
+[*.{yaml, yml}]
+indent_style = space
+indent_size = 2
+
 # 4 space indentation
 [*.py]
 indent_style = space

.github/workflows/validate.yaml

@@ -0,0 +1,77 @@
+name: Validate
+
+on: [push, pull_request, workflow_dispatch]
+
+env:
+  FORCE_COLOR: 1
+  XDG_CACHE_HOME: ${{ github.workspace }}/cache
+
+jobs:
+  validate:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.7", "3.8", "3.9"]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - uses: actions/setup-java@v2
+        with:
+          distribution: "temurin"
+          java-version: "17"
+
+      - name: Get pip cache dir
+        id: pip-cache
+        shell: bash
+        run: |
+          python -m ensurepip --upgrade
+          echo "::set-output name=dir::$(pip cache dir)"
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ matrix.os }}-pip-${{ matrix.python-version }}-v1-${{
+            hashFiles('**/setup.py', '**/requirements*.txt') }}
+          restore-keys: |
+            ${{ matrix.os }}-pip-${{ matrix.python-version }}-v1-
+      - name: Cache xdg
+        uses: actions/cache@v2
+        with:
+          path: ${{ env.XDG_CACHE_HOME }}
+          key: ${{ matrix.os }}-xdg-v1-${{ hashFiles('**/with-fuseki.sh') }}
+          restore-keys: |
+            ${{ matrix.os }}-xdg-v1-
+      - name: Install dependencies
+        shell: bash
+        run: |
+          # bash .travis.fuseki_install_optional.sh
+          pip install --default-timeout 60 -r requirements.txt
+          if [ "${{ matrix.os }}" == "ubuntu-latest" ]
+          then
+              sudo apt-get install -y libdb-dev
+          elif [ "${{ matrix.os }}" == "macos-latest" ]
+          then
+              brew install berkeley-db@4
+              export BERKELEYDB_DIR=$(brew --prefix berkeley-db@4)
+          fi
+          pip install --default-timeout 60 -r requirements.dev.txt
+          pip install networkx
+          python setup.py install
+      - name: Validate
+        shell: bash
+        run: |
+          black --config black.toml --check ./rdflib || true
+          flake8 --exit-zero rdflib
+          if [ "${{ matrix.os }}" == "windows-latest" ]
+          then
+            pytest -ra --cov
+          else
+            ./with-fuseki.sh pytest -ra --cov
+          fi

requirements.dev.txt

@@ -1,4 +1,4 @@
-berkeleydb
+berkeleydb; platform_system != "Windows"
 black==21.9b0
 coverage
 doctest-ignore-unicode==0.1.2

test/test_dataset.py

@@ -1,5 +1,6 @@
 import sys
 import os
+from typing import Optional
 import unittest
 
 from tempfile import mkdtemp, mkstemp
@@ -27,10 +28,13 @@
 
 class DatasetTestCase(unittest.TestCase):
     store = "default"
+    skip_reason = None
     slow = True
     tmppath = None
 
     def setUp(self):
+        if self.skip_reason is not None:
+            self.skipTest(skip_reason)
         try:
             self.graph = Dataset(store=self.store)
         except ImportError:
@@ -207,6 +211,7 @@ def testIter(self):
 tests = 0
 
 for s in plugin.plugins(pluginname, plugin.Store):
+    skip_reason: Optional[str] = None
     if s.name in ("default", "Memory", "Auditable", "Concurrent", "SPARQLStore"):
         continue  # these are tested by default
 
@@ -218,12 +223,14 @@ def testIter(self):
 
         try:
             assert len(urlopen(HOST).read()) > 0
-        except:
-            sys.stderr.write("No SPARQL endpoint for %s (tests skipped)\n" % s.name)
-            continue
+        except BaseException:
+            skip_reason = "No SPARQL endpoint for %s (tests skipped)\n" % s.name
+            sys.stderr.write(skip_reason)
 
     locals()["t%d" % tests] = type(
-        "%sContextTestCase" % s.name, (DatasetTestCase,), {"store": s.name}
+        "%sContextTestCase" % s.name,
+        (DatasetTestCase,),
+        {"store": s.name, "skip_reason": skip_reason},
     )
     tests += 1
 

with-fuseki.sh

@@ -0,0 +1,179 @@
+#!/usr/bin/env bash
+
+# script_name="${0}"
+script_dirname="$( dirname -- "${0}" )"
+# script_basename="$( basename -- "${0}" )"
+
+set -eo pipefail
+
+wait_http_okay() {
+    local timeout=15
+    for var in "${@}"
+    do
+        eval "local ${var}"
+    done
+    1>&2 echo "INFO: checking if ${url} (${label}) is okay"
+    for ((i=1;i<=${timeout};i++))
+    do
+        curl --silent "${url}" --fail -o /dev/null && {
+            1>&2 echo "INFO: url ${url} (${label}) is okay"
+            return 0
+        }
+        kill -0 "${pid}" 2>/dev/null 1>&2 || {
+            1>&2 echo "ERROR: pid ${pid} (${label}) not running anymore ..."
+            return 1
+        }
+        1>&2 echo "INFO: url ${url} (${label}) not okay, waiting ..."
+        sleep 2
+    done
+    1>&2 echo "ERROR: timed out trying to load ${url} (${label})"
+    return 1
+}
+
+assert_pid_running() {
+    for var in "${@}"
+    do
+        eval "local ${var}"
+    done
+
+    kill -0 "${pid}" 2>/dev/null 1>&2 || {
+        1>&2 echo "ERROR: pid ${pid} (${label}) not running anymore ..."
+        return 1
+    }
+    1>&2 echo "INFO: pid ${pid} (${label}) running"
+    return 0
+}
+
+kill_and_wait_pid() {
+    local timeout=10
+    for var in "${@}"
+    do
+        eval "local ${var}"
+    done
+    1>&2 echo "INFO: killing pid ${pid} (${label})"
+    kill "${pid}"
+    for ((i=1;i<=${timeout};i++))
+    do
+        kill -0 "${pid}" 2>/dev/null 1>&2 || {
+            1>&2 echo "INFO: pid ${pid} (${label}) is dead "
+            return 0
+        }
+        1>&2 echo "INFO: pid ${pid} (${label}) is not dead, waiting ..."
+        sleep 2
+    done
+    1>&2 echo "ERROR: wait timeout for pid ${pid} (${label})"
+    return 1
+}
+
+fuseki_pid_normal=""
+fuseki_pid_tdb=""
+
+fuseki_log_normal=""
+fuseki_log_tdb=""
+
+exit_handler() {
+    1>&2 declare -p fuseki_pid_normal fuseki_pid_tdb
+    if [ -z "${FUSEKI_SKIP_SHUTDOWN}" ]
+    then
+        if [ -n "${fuseki_pid_normal}" ]
+        then
+            kill_and_wait_pid "pid=${fuseki_pid_normal}" "label=fuseki-normal" || :
+        fi
+        if [ -n "${fuseki_pid_tdb}" ]
+        then
+            kill_and_wait_pid "pid=${fuseki_pid_tdb}" "label=fuseki-tdb" || :
+        fi
+    else
+        1>&2 echo "WARNING: not killing fuseki fuseki_pid_normal=${fuseki_pid_normal} fuseki_pid_tdb=${fuseki_pid_tdb}"
+    fi
+
+    if [ -n "${FUSEKI_DUMP_LOGS}" ] || [ "${xrc}" != "0" ]
+    then
+        1>&2 echo "dumping fuseki_log_normal=${fuseki_log_normal}"
+        cat "${fuseki_log_normal}" || :
+        1>&2 echo "dumping fuseki_log_tdb=${fuseki_log_tdb}"
+        cat "${fuseki_log_tdb}" || :
+    fi
+}
+
+
+main() {
+    : "${LOCALSTATEDIR:=${script_dirname}/var}"
+    # : ${FUSEKI_PORT:=3030}
+    : "${XDG_CACHE_HOME:=${HOME}/.cache}"
+
+    local jena_uri="https://archive.apache.org/dist/jena/binaries/apache-jena-fuseki-3.17.0.tar.gz"
+    local jena_sha512="2b92f3304743da335f648c1be7b5d7c3a94725ed0a9b5123362c89a50986690114dcef0813e328126b14240f321f740b608cc353417e485c9235476f059bd380"
+    local jena_archive_basename
+    jena_archive_basename="$(basename "${jena_uri}")"
+    local jena_archive="${XDG_CACHE_HOME}/${jena_archive_basename}"
+    local jena_stem="${jena_archive_basename%%.tar.gz}"
+
+    1>&2 declare -p jena_uri jena_archive XDG_CACHE_HOME LOCALSTATEDIR
+    if ! [ -e "${jena_archive}" ]
+    then
+        mkdir -vp "${XDG_CACHE_HOME}"
+        curl "${jena_uri}" -o "${jena_archive}"
+    fi
+    printf "%s  %s\n" "${jena_sha512}" "${jena_archive_basename}" > "${jena_archive}.sha512"
+    (cd "${XDG_CACHE_HOME}" && shasum -c -a512 "${jena_archive_basename}.sha512") || {
+        echo 1>&2 "ERROR: digest verification failed"
+        rm -rv "${jena_archive}" "${jena_archive}.sha512"
+    }
+    mkdir -vp "${LOCALSTATEDIR}"
+    tar -zxf "${jena_archive}" -C "${LOCALSTATEDIR}"
+
+    local FUSEKI_HOME=${LOCALSTATEDIR}/${jena_stem}
+    1>&2 declare -p FUSEKI_HOME
+    export FUSEKI_HOME
+
+    fuseki_base_normal="${FUSEKI_HOME}/run-normal"
+    fuseki_base_tdb="${FUSEKI_HOME}/run-tdb"
+    local fuseki_pidfile_normal="${fuseki_base_normal}/server.pid"
+    local fuseki_pidfile_tdb="${fuseki_base_tdb}/server.pid"
+
+    mkdir -vp "${fuseki_base_normal}" "${fuseki_base_tdb}"
+    local fuseki_port_normal=3030
+    local fuseki_port_tdb=3031
+
+    1>&2 declare -p fuseki_base_normal fuseki_base_tdb fuseki_pidfile_normal fuseki_pidfile_tdb fuseki_port_normal fuseki_port_tdb
+
+    trap exit_handler EXIT
+
+    fuseki_log_normal="${fuseki_base_normal}/out.log"
+    fuseki_log_tdb="${fuseki_base_tdb}/out.log"
+
+    1>&2 declare -p fuseki_log_normal fuseki_log_tdb
+
+    1>&2 echo "INFO: starting fuseki: normal"
+    FUSEKI_BASE="${fuseki_base_normal}" bash "${FUSEKI_HOME}/fuseki-server" \
+        --port "${fuseki_port_normal}" --debug \
+        --update --mem /db &>"${fuseki_log_normal}" &
+    fuseki_pid_normal="${!}"
+    echo "${fuseki_pid_normal}" > "${fuseki_pidfile_normal}"
+
+    1>&2 echo "INFO: starting fuseki: tdb"
+    FUSEKI_BASE="${fuseki_base_tdb}" bash "${FUSEKI_HOME}/fuseki-server" \
+        --port "${fuseki_port_tdb}" --debug \
+        --update --memTDB --set tdb:unionDefaultGraph=true /db &>"${fuseki_log_tdb}" &
+    fuseki_pid_tdb="${!}"
+    echo "${fuseki_pid_tdb}" > "${fuseki_pidfile_tdb}"
+
+    1>&2 declare -p fuseki_pid_tdb fuseki_pid_normal
+
+    wait_http_okay "pid=${fuseki_pid_normal}" "url=http://localhost:${fuseki_port_normal}/" "label=fuseki-normal"
+    wait_http_okay "pid=${fuseki_pid_tdb}" "url=http://localhost:${fuseki_port_tdb}/" "label=fuseki-tdb"
+
+    assert_pid_running "pid=${fuseki_pid_tdb}" "label=fuseki-tdb"
+    assert_pid_running "pid=${fuseki_pid_normal}" "label=fuseki-normal"
+
+    local -a args=("${@}")
+
+    1>&2 echo "running: ${args[*]}"
+
+    local xrc=1
+    "${args[@]}"; xrc="${?}"
+    exit "${xrc}"
+}
+
+main "${@}"