Merge pull request #84 from SaftIng/feature/configureTravisForVirtuoso

Setup travis for virtuoso
SaftIng · Dec 19, 2016 · af0d027 · af0d027
2 parents fcdcf06 + e74a137
commit af0d027
Show file tree

Hide file tree

Showing 5 changed files with 378 additions and 2 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -5,24 +5,48 @@ php:
     - '5.5'
     - '5.6'
     - '7.0'
-    - hhvm
+    - '7.1'
     - nightly
 
+env:
+    - VIRTUOSO=6.1.7
+    - VIRTUOSO=7.1.0
+    - VIRTUOSO=7.2.4.2
+
 matrix:
+    fast_finish: true
+    include:
+        - php: hhvm
+        - php: '5.4'
+          env: VIRTUOSO=6.1.4
+        - php: '5.4'
+          env: VIRTUOSO=6.1.6
+        - php: '5.4'
+          env: VIRTUOSO=7.2.0
     allow_failures:
+        - env: VIRTUOSO=6.1.4
+        - env: VIRTUOSO=6.1.6
         - php: nightly
 
-sudo: false
+cache:
+    directories:
+        - $HOME/.composer/cache
+        - virtuoso-opensource
+
+sudo: true
 
 services:
     - mysql
 
 before_install:
+    - bash ./scripts/travis/install-extensions.sh
+    - bash ./scripts/travis/install-services.sh
     - composer require satooshi/php-coveralls:1.*
     - travis_retry composer install --dev --no-interaction
 
 before_script:
     - cp test-config.yml.travis test-config.yml
+    - sed -i 's|<[/]\?log[^>]*>||' phpunit.xml
 
 script: vendor/bin/phpunit --coverage-clover gen/coverage/clover.xml
 

diff --git a/scripts/travis/README.md b/scripts/travis/README.md
@@ -0,0 +1,79 @@
+- `install-services.sh` to handle the install of additional services
+
+## SPARQL services
+
+This file and scripts are taken form the SemanticMediaWiki repo and only selected files where copied for running tests with virtuoso.
+It is taken as of commit 721a63d3a73400300f73e0a088196a6ed0fe5afd from https://github.com/SemanticMediaWiki/SemanticMediaWiki.
+The unsupported services are marked with *Endabled* "No".
+
+<table>
+	<tr>
+		<th>Endabled</th>
+		<th>Service</th>
+		<th>Connector</th>
+		<th>QueryEndPoint</th>
+		<th>UpdateEndPoint</th>
+		<th>DataEndpoint</th>
+		<th>DefaultGraph</th>
+		<th>Comments</th>
+	</tr>
+	<tr>
+		<th>No</th>
+		<th>Fuseki (mem)<sup>1</sup></th>
+		<td>Fuseki</td>
+		<td>http://localhost:3030/db/query</td>
+		<td>http://localhost:3030/db/update</td>
+		<td>''</td>
+		<td>''</td>
+		<td>fuseki-server --update --port=3030 --mem /db</td>
+	</tr>
+	<tr>
+		<th>No</th>
+		<th>Fuseki (memTDB)</th>
+		<td>Fuseki</td>
+		<td>http://localhost:3030/db/query</td>
+		<td>http://localhost:3030/db/update</td>
+		<td>''</td>
+		<td>http://example.org/myFusekiGraph</td>
+		<td>fuseki-server --update --port=3030 --memTDB --set tdb:unionDefaultGraph=true /db</td>
+	</tr>
+	<tr>
+		<th>Yes</th>
+		<th>Virtuoso opensource</th>
+		<td>Virtuoso</td>
+		<td>http://localhost:8890/sparql</td>
+		<td>http://localhost:8890/sparql</td>
+		<td>''</td>
+		<td>http://example.org/myVirtuosoGraph</td>
+		<td>sudo apt-get install virtuoso-opensource</td>
+	</tr>
+	<tr>
+		<th>No</th>
+		<th>4store<sup>2</sup></th>
+		<td>4store</td>
+		<td>http://localhost:8088/sparql/</td>
+		<td>http://localhost:8088/update/</td>
+		<td>''</td>
+		<td>http://example.org/myFourGraph</td>
+		<td>apt-get install 4store</td>
+	</tr>
+	<tr>
+		<th>No</th>
+		<th>Sesame</th>
+		<td>Custom</td>
+		<td>http://localhost:8080/openrdf-sesame/repositories/test-smw</td>
+		<td>http://localhost:8080/openrdf-sesame/repositories/test-smw/statements</td>
+		<td>''</td>
+		<td>`test-smw` is specifed as native in-memory store</td>
+		<td></td>
+	</tr>
+
+</table>
+
+<sup>1</sup> When running integration tests with [Jena Fuseki][fuseki] it is suggested that the `in-memory` option is used to avoid potential loss of production data during test execution.
+
+<sup>2</sup> Currently, Travis-CI doesn't support `4Store` (1.1.4-2) as service but the following configuration has been sucessfully tested with the available test suite. ([issue #110](https://github.com/garlik/4store/issues/110) )
+
+[fuseki]: https://jena.apache.org/
+[virtuoso]: https://github.com/openlink/virtuoso-opensource
+[4store]: https://github.com/garlik/4store
diff --git a/scripts/travis/install-extensions.sh b/scripts/travis/install-extensions.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+echo $TRAVIS_PHP_VERSION
+
+# skip hhvm
+if [[ $TRAVIS_PHP_VERSION = "hhv"* ]]; then
+    exit 0
+fi
+
+# get build dependencies
+sudo apt-get install -y unixODBC-dev
+
+PHPVERSION=$( php -v | head -n1 | sed "s|^PHP \([0-9][0-9\.]*\).*$|\1|" | tr -d '\n' )
+
+ls ~/.phpenv/versions/
+echo "PHPVERSION: " $PHPVERSION
+echo "LOADED CONFIG: " `php --ini | grep "Loaded Configuration" | sed -e "s|.*:\s*||"`
+
+# get php sources
+wget https://github.com/php/php-src/archive/php-$PHPVERSION.tar.gz
+ls
+tar -xzf php-$PHPVERSION.tar.gz
+
+# build odbc extension
+cd php-src-php-$PHPVERSION/ext/odbc/
+phpize
+# use fix from https://github.com/docker-library/php/issues/103
+sed -ri 's@^ *test +"\$PHP_.*" *= *"no" *&& *PHP_.*=yes *$@#&@g' configure
+./configure --with-unixODBC=shared,/usr
+make
+make install
+
+# enable odbc
+echo "extension=odbc.so" >> `php --ini | grep "Loaded Configuration" | sed -e "s|.*:\s*||"`
+
+# build pdo_odbc
+cd ../pdo_odbc/
+phpize
+./configure --with-pdo-odbc=unixODBC,/usr
+make
+make install
+
+#enable pdo_odbc
+echo "extension=pdo_odbc.so" >> `php --ini | grep "Loaded Configuration" | sed -e "s|.*:\s*||"`
+php -m
diff --git a/scripts/travis/install-services.sh b/scripts/travis/install-services.sh
@@ -0,0 +1,222 @@
+#!/bin/bash
+set -ex
+BASE_PATH=$(pwd)
+E_UNREACHABLE=86
+
+# skip hhvm
+if [[ $TRAVIS_PHP_VERSION = "hhv"* ]]; then
+    exit 0
+fi
+
+if [ "$FOURSTORE" != "" ] || [ "$VIRTUOSO" != "" ] || [ "$SESAME" != "" ] || [[ "$FUSEKI" == "2."* ]]
+then
+	sudo apt-get update -qq
+fi
+
+# Version 1.1.0 is available and testable on Travis/SMW
+if [ "$FUSEKI" != "" ]
+then
+	# Archive
+	# http://archive.apache.org/dist/jena/binaries/jena-fuseki-$FUSEKI-distribution.tar.gz
+	# http://www.eu.apache.org/dist/jena/binaries/jena-fuseki-$FUSEKI-distribution.tar.gz
+
+	# Avoid ERROR 503: Service Unavailable
+	# wget http://archive.apache.org/dist/jena/binaries/jena-fuseki-$FUSEKI-distribution.tar.gz
+
+	if [[ "$FUSEKI" == "2."* ]]
+	then
+
+		# Fuseki requires Java8 for Fuseki2 v2.3.0 onwards
+		sudo apt-get install oracle-java8-installer
+
+		export JAVA_HOME="/usr/lib/jvm/java-8-oracle";
+		export PATH="$PATH:/usr/lib/jvm/java-8-oracle/bin";
+		export java_path="/usr/lib/jvm/java-8-oracle/jre/bin/java";
+
+		wget https://github.com/mwjames/travis-support/raw/master/fuseki/$FUSEKI/apache-jena-fuseki-$FUSEKI.tar.gz
+
+		# option z caused "gzip: stdin: not in gzip format"
+		tar -xf apache-jena-fuseki-$FUSEKI.tar.gz
+		mv apache-jena-fuseki-$FUSEKI fuseki
+	else
+		wget https://github.com/mwjames/travis-support/raw/master/fuseki/$FUSEKI/jena-fuseki-$FUSEKI-distribution.tar.gz
+
+		tar -zxf jena-fuseki-$FUSEKI-distribution.tar.gz
+		mv jena-fuseki-$FUSEKI fuseki
+	fi
+
+	cd fuseki
+
+	## Start fuseki in-memory as background
+	bash fuseki-server --update --mem /db &>/dev/null &
+fi
+
+if [ "$SESAME" != "" ]
+then
+	TOMCAT_VERSION=tomcat6
+	sudo java -version
+
+	sudo apt-get install $TOMCAT_VERSION
+
+	CATALINA_BASE=/var/lib/$TOMCAT_VERSION
+	CATALINA_HOME=/usr/share/$TOMCAT_VERSION
+
+	sudo chown $USER -R $CATALINA_BASE/
+	sudo chmod g+rw -R $CATALINA_BASE/
+
+	sudo mkdir -p $CATALINA_HOME/.aduna
+	sudo chown -R $TOMCAT_VERSION:$TOMCAT_VERSION $CATALINA_HOME
+
+	# One method to get the war files
+	# wget http://search.maven.org/remotecontent?filepath=org/openrdf/sesame/sesame-http-server/$SESAME/sesame-http-server-$SESAME.war -O openrdf-sesame.war
+	# wget http://search.maven.org/remotecontent?filepath=org/openrdf/sesame/sesame-http-workbench/$SESAME/sesame-http-workbench-$SESAME.war -O openrdf-workbench.war
+	# cp *.war /var/lib/tomcat6/webapps/
+
+	# http://sourceforge.net/projects/sesame/
+	# Unreliable sourceforge.net download
+	# wget http://downloads.sourceforge.net/project/sesame/Sesame%202/$SESAME/openrdf-sesame-$SESAME-sdk.zip
+	wget https://github.com/mwjames/travis-support/raw/master/sesame/$SESAME/openrdf-sesame-$SESAME-sdk.zip
+
+	# tar caused a lone zero block, using zip instead
+	unzip -q openrdf-sesame-$SESAME-sdk.zip
+	cp openrdf-sesame-$SESAME/war/*.war $CATALINA_BASE/webapps/
+
+	sudo service $TOMCAT_VERSION restart
+	ps -ef | grep tomcat
+
+	sleep 5
+
+	if curl --output /dev/null --silent --head --fail "http://localhost:8080/openrdf-sesame"
+	#if curl --output /dev/null --silent --head --fail "http://localhost:8080/openrdf-sesame/home/overview.view"
+	then
+		echo "openrdf-sesame service url is reachable"
+	else
+		echo "openrdf-sesame service url is not reachable"
+		sudo cat $CATALINA_BASE/logs/*.log &
+		sudo cat $CATALINA_BASE/logs/catalina.out &
+		exit $E_UNREACHABLE
+	fi
+
+	./openrdf-sesame-$SESAME/bin/console.sh < $BASE_PATH/scripts/travis/openrdf-sesame-memory-repository.txt
+fi
+
+# Version 1.1.4-1 is available but has a problem
+# https://github.com/garlik/4store/issues/110
+# 4STORE can not be used as variable name therefore FOURSTORE
+if [ "$FOURSTORE" != "" ]
+then
+
+	sudo mkdir /var/lib/4store/
+	sudo mkdir /var/lib/4store/db
+	sudo chown $USER -R /var/lib/4store/
+	sudo chmod g+rw -R /var/lib/4store/
+
+	sudo apt-get install 4store=$FOURSTORE
+
+	## Disabling the firewall
+	sudo iptables -F
+
+	4s-backend-setup db
+	4s-backend db
+
+	## Output the current process table
+	ps auwwx | grep 4s-
+
+	## -D only used to check the status of the 4store instance
+	## 4s-httpd -D -p 8088 db
+
+	4s-httpd -p 8088 db
+fi
+
+# We build all Virtuoso version from scratch
+if [[ "$VIRTUOSO" != "" ]]
+then
+	sudo apt-get install libssl-dev -q
+	sudo apt-get install autoconf automake bison flex gawk gperf libtool -q
+
+	if [[ -f virtuoso-opensource/$VIRTUOSO/binsrc/virtuoso/virtuoso-t ]]
+	then
+		echo "use cached virtuoso-opensource"
+		cd virtuoso-opensource/$VIRTUOSO
+	else
+		#git clone git://github.com/openlink/virtuoso-opensource.git
+		#cd virtuoso-opensource
+		#git pull origin stable/7
+		wget --no-check-certificate -q https://github.com/openlink/virtuoso-opensource/archive/v$VIRTUOSO.zip -O virtuoso-opensource.zip
+
+		unzip -q virtuoso-opensource.zip
+		rm -r virtuoso-opensource/$VIRTUOSO || true
+		mv virtuoso-opensource-$VIRTUOSO virtuoso-opensource/$VIRTUOSO
+
+		cd virtuoso-opensource/$VIRTUOSO
+		./autogen.sh
+
+		# --disable-all-vads: This parameter disables building all the VAD packages (tutorials, demos, etc.).
+		# --with-readline: This parameter is used so that the system Readline library is used
+		# --program-transform-name: Both Virtuoso and unixODBC install a program named isql. Use this parameter to rename virtuosos program to isql-v
+
+		./configure --program-transform-name="s/isql/isql-v/" --with-readline --disable-all-vads |& tee #configure.log
+
+		# Only output error and warnings
+		make > /dev/null
+	fi
+
+	# Build tree to start the automated test suite
+	# make check
+
+	sudo make install
+
+	## For Virtuoso
+	#export PATH=$PATH:/usr/local/virtuoso-opensource/bin
+
+	sudo /usr/local/virtuoso-opensource/bin/virtuoso-t -f -c /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.ini &
+	#sudo /usr/local/virtuoso-opensource/bin/virtuoso-t -f &
+
+	sleep 15
+
+	sudo /usr/local/virtuoso-opensource/bin/isql-v 1111 dba dba $BASE_PATH/scripts/travis/virtuoso-sparql-permission.sql
+
+	# configure datasource name for ODBC connection
+	echo "[VOS]" | sudo tee -a /etc/odbc.ini > /dev/null
+	echo "Driver=/usr/local/virtuoso-opensource/lib/virtodbc.so" | sudo tee -a /etc/odbc.ini > /dev/null
+	echo "Description=Virtuoso OpenSource Edition" | sudo tee -a /etc/odbc.ini > /dev/null
+	echo "Address=localhost:1111" | sudo tee -a /etc/odbc.ini > /dev/null
+fi
+
+#@see  http://wiki.blazegraph.com/wiki/index.php/NanoSparqlServer
+if [ "$BLAZEGRAPH" != "" ]
+then
+	#sudo apt-get install tomcat6
+
+	#sudo chown $USER -R /var/lib/tomcat6/
+	#sudo chmod g+rw -R /var/lib/tomcat6/
+
+	#sudo mkdir -p /usr/share/tomcat6/.aduna
+	#sudo chown -R tomcat6:tomcat6 /usr/share/tomcat6
+
+	# http://sourceforge.net/projects/bigdata/
+	#wget http://downloads.sourceforge.net/project/bigdata/bigdata/$BLAZEGRAPH/bigdata.war
+
+	#cp bigdata.war /var/lib/tomcat6/webapps/
+	#export JAVA_OPTS="-server -Xmx2g -Dcom.bigdata.rdf.sail.webapp.ConfigParams.propertyFile="$BASE_PATH/scripts/travis/blazegraph-store.properties
+
+	#sudo service tomcat6 restart
+	#sleep 3
+
+	#Using the jar
+	# Unreliable sourceforge.net download
+	# wget http://downloads.sourceforge.net/project/bigdata/bigdata/$BLAZEGRAPH/bigdata-bundled.jar
+	wget https://github.com/mwjames/travis-support/raw/master/blazegraph/$BLAZEGRAPH/bigdata-bundled.jar
+
+	java -server -Xmx4g -Dbigdata.propertyFile=$BASE_PATH/scripts/travis/blazegraph-store.properties -jar bigdata-bundled.jar &>/dev/null &
+	sleep 5
+
+	if curl --output /dev/null --silent --head --fail "http://localhost:9999/bigdata"
+	then
+		echo "blazegraph service url is reachable"
+	else
+		echo "blazegraph service url is not reachable"
+		exit $E_UNREACHABLE
+	fi
+
+fi