deploy version 0.3.39

Change-Id: Ieecaf2226634609b3f8169067af7e589afefe97a

blazegraph-service-0.3.39.war

@@ -0,0 +1 @@
+#$# git-fat 266362107ae2a38125f730c00bb9f6f46f8f66f9             79797263

forAllCategoryWikis.sh

@@ -12,7 +12,7 @@ else
 fi
 
 shift
-$fetch $DUMP_LIST | while read wiki; do
+$fetch $DUMP_LIST | grep -v '^#' | while read wiki; do
 	echo "Processing $wiki..."
 	$DIR/$COMMAND $wiki "$@"
 done

jetty-runner-9.4.12.v20180830.jar

@@ -0,0 +1 @@
+#$# git-fat 55ffeddc2d9153f726249984d7f3efa2a23649da              7074499

lib/wikidata-query-tools-0.3.39-jar-with-dependencies.jar

@@ -0,0 +1 @@
+#$# git-fat c8e9c5b0d079a85f06dc6a54c2767e399f4e548b             14152823

prefixes-sdc.conf

@@ -0,0 +1,51 @@
+PREFIX psn: <http://www.wikidata.org/prop/statement/value-normalized/>
+PREFIX pqn: <http://www.wikidata.org/prop/qualifier/value-normalized/>
+PREFIX prn: <http://www.wikidata.org/prop/reference/value-normalized/>
+PREFIX mwapi: <https://www.mediawiki.org/ontology#API/>
+PREFIX gas: <http://www.bigdata.com/rdf/gas#>
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX owl: <http://www.w3.org/2002/07/owl#>
+PREFIX wikibase: <http://wikiba.se/ontology#>
+PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
+PREFIX schema: <http://schema.org/>
+PREFIX cc: <http://creativecommons.org/ns#>
+PREFIX geo: <http://www.opengis.net/ont/geosparql#>
+PREFIX prov: <http://www.w3.org/ns/prov#>
+PREFIX wd: <http://commons.wikimedia.org/entity/>
+PREFIX data: <https://commons.wikimedia.org/wiki/Special:EntityData/>
+PREFIX s: <http://commons.wikimedia.org/entity/statement/>
+PREFIX ref: <http://commons.wikimedia.org/reference/>
+PREFIX v: <http://commons.wikimedia.org/value/>
+PREFIX wdt: <http://commons.wikimedia.org/prop/direct/>
+PREFIX wdtn: <http://commons.wikimedia.org/prop/direct-normalized/>
+PREFIX p: <http://commons.wikimedia.org/prop/>
+PREFIX ps: <http://commons.wikimedia.org/prop/statement/>
+PREFIX psv: <http://commons.wikimedia.org/prop/statement/value/>
+PREFIX psn: <http://commons.wikimedia.org/prop/statement/value-normalized/>
+PREFIX pq: <http://commons.wikimedia.org/prop/qualifier/>
+PREFIX pqv: <http://commons.wikimedia.org/prop/qualifier/value/>
+PREFIX pqn: <http://commons.wikimedia.org/prop/qualifier/value-normalized/>
+PREFIX pr: <http://commons.wikimedia.org/prop/reference/>
+PREFIX prv: <http://commons.wikimedia.org/prop/reference/value/>
+PREFIX prn: <http://commons.wikimedia.org/prop/reference/value-normalized/>
+PREFIX wdno: <http://commons.wikimedia.org/prop/novalue/>
+PREFIX d: <http://www.wikidata.org/entity/>
+PREFIX ddata: <https://www.wikidata.org/wiki/Special:EntityData/>
+PREFIX ds: <http://www.wikidata.org/entity/statement/>
+PREFIX dref: <http://www.wikidata.org/reference/>
+PREFIX dv: <http://www.wikidata.org/value/>
+PREFIX dt: <http://www.wikidata.org/prop/direct/>
+PREFIX dtn: <http://www.wikidata.org/prop/direct-normalized/>
+PREFIX dp: <http://www.wikidata.org/prop/>
+PREFIX dps: <http://www.wikidata.org/prop/statement/>
+PREFIX dpsv: <http://www.wikidata.org/prop/statement/value/>
+PREFIX dpsn: <http://www.wikidata.org/prop/statement/value-normalized/>
+PREFIX dpq: <http://www.wikidata.org/prop/qualifier/>
+PREFIX dpqv: <http://www.wikidata.org/prop/qualifier/value/>
+PREFIX dpqn: <http://www.wikidata.org/prop/qualifier/value-normalized/>
+PREFIX dpr: <http://www.wikidata.org/prop/reference/>
+PREFIX dprv: <http://www.wikidata.org/prop/reference/value/>
+PREFIX dprn: <http://www.wikidata.org/prop/reference/value-normalized/>
+PREFIX dno: <http://www.wikidata.org/prop/novalue/>

runBlazegraph.sh

@@ -10,6 +10,9 @@ HOST=${HOST:-"localhost"}
 CONTEXT=bigdata
 PORT=${PORT:-"9999"}
 DIR=${DIR:-`dirname $0`}
+PREFIXES_FILE=$DIR/prefixes.conf
+WIKIBASE_CONCEPT_URI_PARAM=${WIKIBASE_CONCEPT_URI_PARAM:-""}
+COMMONS_CONCEPT_URI_PARAM=${COMMONS_CONCEPT_URI_PARAM:-""}
 HEAP_SIZE=${HEAP_SIZE:-"16g"}
 LOG_CONFIG=${LOG_CONFIG:-""}
 LOG_DIR=${LOG_DIR:-"/var/log/wdqs"}
@@ -47,11 +50,12 @@ DEBUG=-agentlib:jdwp=transport=dt_socket,server=y,address=8000,suspend=n
 DEBUG=
 
 function usage() {
-  echo "Usage: $0 [-h <host>] [-d <dir>] [-c <context>] [-p <port>] [-o <blazegraph options>] [-f config.properties]"
+  echo "Usage: $0 [-h <host>] [-d <dir>] [-c <context>] [-p <port>] " \
+  "[-o <blazegraph options>] [-f config.properties] [-n prefixes.conf] [-w wikibaseConceptUri] [-m commonsConceptUri]"
   exit 1
 }
 
-while getopts h:c:p:d:o:f:? option
+while getopts h:c:p:d:o:f:n:w:m:? option
 do
   case "${option}"
   in
@@ -61,6 +65,9 @@ do
     d) DIR=${OPTARG};;
     o) BLAZEGRAPH_OPTS="${OPTARG}";;
     f) CONFIG_FILE=${OPTARG};;
+    n) PREFIXES_FILE=${OPTARG};;
+    w) WIKIBASE_CONCEPT_URI_PARAM="-DwikibaseConceptUri=${OPTARG} ";;
+    m) COMMONS_CONCEPT_URI_PARAM="-DcommonsConceptUri=${OPTARG} ";;
     ?) usage;;
   esac
 done
@@ -80,17 +87,19 @@ fi
 echo "Running Blazegraph from `pwd` on :$PORT/$CONTEXT"
 exec java \
      -server -XX:+UseG1GC ${MEMORY} ${DEBUG} ${GC_LOGS} ${LOG_OPTIONS} ${EXTRA_JVM_OPTS} \
-     -Dcom.bigdata.rdf.sail.webapp.ConfigParams.propertyFile=${CONFIG_FILE} \
+     -Dcom.bigdata.rdf.sail.webapp.ConfigParams.propertyFile="${CONFIG_FILE}" \
      -Dorg.eclipse.jetty.server.Request.maxFormContentSize=200000000 \
      -Dcom.bigdata.rdf.sparql.ast.QueryHints.analytic=true \
      -Dcom.bigdata.rdf.sparql.ast.QueryHints.analyticMaxMemoryPerQuery=1073741824 \
      -DASTOptimizerClass=org.wikidata.query.rdf.blazegraph.WikibaseOptimizers \
      -Dorg.wikidata.query.rdf.blazegraph.inline.literal.WKTSerializer.noGlobe=$DEFAULT_GLOBE \
      -Dcom.bigdata.rdf.sail.webapp.client.RemoteRepository.maxRequestURLLength=7168 \
-     -Dcom.bigdata.rdf.sail.sparql.PrefixDeclProcessor.additionalDeclsFile=$DIR/prefixes.conf \
+     -Dcom.bigdata.rdf.sail.sparql.PrefixDeclProcessor.additionalDeclsFile="$PREFIXES_FILE" \
      -Dorg.wikidata.query.rdf.blazegraph.mwapi.MWApiServiceFactory.config=$DIR/mwservices.json \
      -Dcom.bigdata.rdf.sail.webapp.client.HttpClientConfigurator=org.wikidata.query.rdf.blazegraph.ProxiedHttpConnectionFactory \
      -Dhttp.userAgent="${USER_AGENT}" \
+     $WIKIBASE_CONCEPT_URI_PARAM \
+     $COMMONS_CONCEPT_URI_PARAM \
      -Dorg.eclipse.jetty.annotations.AnnotationParser.LEVEL=OFF \
      ${BLAZEGRAPH_OPTS} \
      -jar jetty-runner*.jar \

runStreamingUpdater.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+set -e
+
+if [ -r /etc/default/wdqs-streaming-updater ]; then
+  . /etc/default/wdqs-streaming-updater
+fi
+
+HOST=http://localhost:9999
+CONTEXT=bigdata
+HEAP_SIZE=${HEAP_SIZE:-"1g"}
+MEMORY=${MEMORY:-"-Xmx${HEAP_SIZE}"}
+LOG_DIR=${LOG_DIR:-"/var/log/wdqs"}
+if [[ "$JAVA_MAJOR_VERSION" -ge "9" ]] ; then
+    GC_LOGS=${GC_LOGS:-"-Xlog:gc:${LOG_DIR}/wdqs-streaming-updater_jvm_gc.%p.log \
+         -Xlog:gc* \
+         -XX:+UnlockExperimentalVMOptions \
+         -XX:G1NewSizePercent=20 \
+         -XX:+ParallelRefProcEnabled"}
+else
+    GC_LOGS=${GC_LOGS:-"-Xloggc:${LOG_DIR}/wdqs-streaming-updater_jvm_gc.%p.log \
+         -XX:+PrintGCDetails \
+         -XX:+PrintGCDateStamps \
+         -XX:+PrintGCTimeStamps \
+         -XX:+PrintAdaptiveSizePolicy \
+         -XX:+PrintReferenceGC \
+         -XX:+PrintGCCause \
+         -XX:+PrintGCApplicationStoppedTime \
+         -XX:+PrintTenuringDistribution \
+         -XX:+UseGCLogFileRotation \
+         -XX:NumberOfGCLogFiles=10 \
+         -XX:GCLogFileSize=20M"}
+fi
+EXTRA_JVM_OPTS=${EXTRA_JVM_OPTS:-""}
+LOG_CONFIG=${LOG_CONFIG:-""}
+NAMESPACE=wdq
+UPDATER_OPTS=${UPDATER_OPTS:-""}
+
+while getopts h:c:n:t option
+do
+  case "${option}"
+  in
+    h) HOST=${OPTARG};;
+    c) CONTEXT=${OPTARG};;
+    n) NAMESPACE=${OPTARG};;
+    t) TMO=${OPTARG};;
+  esac
+done
+
+# allow extra args
+shift $((OPTIND-1))
+
+if [ -z "$NAMESPACE" ]
+then
+  echo "Usage: $0 -n <namespace> [-h <host>] [-c <context>] [-S] [-v]"
+  exit 1
+fi
+
+if [ -z "$TMO" ]; then
+    TIMEOUT_ARG=
+else
+    TIMEOUT_ARG="-Dorg.wikidata.query.rdf.tool.rdf.RdfRepositoryUpdater.timeout=$TMO"
+fi
+
+LOG_OPTIONS=""
+if [ ! -z "$LOG_CONFIG" ]; then
+    LOG_OPTIONS="-Dlogback.configurationFile=${LOG_CONFIG}"
+fi
+
+CP=`dirname $BASH_SOURCE`/lib/wikidata-query-tools-*-jar-with-dependencies.jar
+MAIN=org.wikidata.query.rdf.tool.StreamingUpdate
+SPARQL_URL=$HOST/$CONTEXT/namespace/$NAMESPACE/sparql
+echo "Updating via $SPARQL_URL"
+exec java -cp ${CP} ${MEMORY} ${GC_LOGS} ${LOG_OPTIONS} ${EXTRA_JVM_OPTS} \
+     ${TIMEOUT_ARG} ${UPDATER_OPTS} \
+     ${MAIN} --sparqlUrl ${SPARQL_URL} "$@"