Skip to content

Commit

Permalink
RYA-495 Closes #297, Updated the Vagrantfile's dependencies
Browse files Browse the repository at this point in the history
  • Loading branch information
@ejwhite922
ejwhite922 authored and Valiyil committed Jun 4, 2018
1 parent 70546b5 commit 3c3ab0d
Show file tree
Hide file tree
Showing 2 changed files with 111 additions and 104 deletions.
203 changes: 105 additions & 98 deletions extras/vagrantExample/src/main/vagrant/Vagrantfile
View file
Expand Up @@ -39,33 +39,35 @@ Vagrant.configure(2) do |config|
config.vm.provider "virtualbox" do |vb|
vb.name = "rya-example-box"
vb.memory = "4096"
vb.cpus = 4
vb.cpus = 4
end

config.vm.network :private_network, ip: "192.168.33.10"
config.vm.hostname = "rya-example-box"

config.vm.provision "shell", inline: <<-SHELL
###set -x ## turn on command echo with expanded variables
# List of dependency versions
export ACCUMULO_VERSION=1.6.5
###export ACCUMULO_VERSION=1.7.1
export HADOOP_VERSION=2.7.2
export RYA_EXAMPLE_VERSION=4.0.0-incubating
export RYA_EXAMPLE_VERSION=4.0.0-incubating-SNAPSHOT
# TODO: Eventually this version of the Rya distribution will be on maven...and then we can use the following instead
#export RYA_EXAMPLE_VERSION=4.0.0-incubating
export RDF4J_VERSION=2.3.1
export ZOOKEEPER_VERSION=3.4.5-cdh4.5.0
mavenRepoUrl=http://repo1.maven.org/maven2/
echo "Updating host file with permanent ip"
sudo sed -i 's/127.0.1.1/192.168.33.10/' /etc/hosts
sudo sed -i 's/127.0.1.1/192.168.33.10/' /etc/hosts
cat >> /etc/hosts <<EOF
192.168.33.10 zoo1 zoo2 zoo3
EOF
sudo -E apt-get -qq update
echo "Installing Java installer..."
sudo -E add-apt-repository ppa:webupd8team/java || exit $?
sudo -E apt-get -qq update || exit $?
Expand All @@ -79,10 +81,10 @@ EOF
echo "Installing Java..."
sudo -E apt-get -qq install -y oracle-java8-installer || exit $?
sudo ln --force -s /usr/lib/jvm/java-8-oracle/ /usr/lib/jvm/default-java
echo "Installing Tomcat..."
sudo -E apt-get install -y tomcat7 || exit $?
echo "Installing Unzip..."
apt-get install unzip || exit $?
Expand Down Expand Up @@ -130,16 +132,16 @@ EOF
### command to list the 7 correct java processes: tomcat-catalina, zookeeper, and 5 Accumulo: tracer, master, monitor, tserver, gc.
function ryaps() { ps -ef | grep java | tr ' ' '\\n' | egrep '^org\\.apache|^tracer|^master|^monitor|^tserver|^gc' | sed '/\\.Main/ N ; s/\\n/ /' ; }
EOF
source ${ACCUMULO_RC} || exit 151
# include it at the beginning of both shell configuration files.
for BASHRC in /home/vagrant/.bashrc /home/vagrant/.bash_profile ;
source ${ACCUMULO_RC} || exit 151
# include it at the beginning of both shell configuration files.
for BASHRC in /home/vagrant/.bashrc /home/vagrant/.bash_profile ;
do touch ${BASHRC} ;
cat - ${BASHRC} > ${BASHRC}.new <<EOF && mv ${BASHRC}.new ${BASHRC} || exit 152
source ${ACCUMULO_RC}
EOF
done
done
echo "Acquiring and Extracting ..."
function echoerr() { printf "%s\n" "$*" >&2; }
Expand Down Expand Up @@ -172,9 +174,9 @@ EOF
fi
done
}
echo "- Hadoop"
hadoopUrl=http://apache.mirrors.tds.net/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz
hadoopUrl=https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz
if [[ ! -d ${HADOOP_HOME} ]] ; then
echo "Downloading $hadoopUrl"
download $hadoopUrl \
Expand All @@ -190,13 +192,13 @@ EOF
fi
echo "- Accumulo"
accumuloUrl=http://apache.mirrors.pair.com/accumulo/${ACCUMULO_VERSION}/accumulo-${ACCUMULO_VERSION}-bin.tar.gz
if [[ ! -d ${ACCUMULO_HOME} ]] ; then
echo "Downloading $accumuloUrl"
download $accumuloUrl \
| tar -zxC /home/vagrant || exit 103
fi
accumuloUrl=https://archive.apache.org/dist/accumulo/${ACCUMULO_VERSION}/accumulo-${ACCUMULO_VERSION}-bin.tar.gz
if [[ ! -d ${ACCUMULO_HOME} ]] ; then
echo "Downloading $accumuloUrl"
download $accumuloUrl \
| tar -zxC /home/vagrant || exit 103
fi
echo "Configuring Zookeeper..."
sudo mkdir --parents /var/zookeeper
sudo chown vagrant:vagrant /var/zookeeper
Expand All @@ -206,15 +208,15 @@ EOF
# Conflicts with Accumulo and maybe Zookeeper
sudo rm --force ${HADOOP_HOME}/share/hadoop/common/lib/slf4j-api-1.7.10.jar
sudo rm --force ${HADOOP_HOME}/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar
# Assure logs are creatable and writeable
sudo mkdir --parents ${ZOO_LOG_DIR}
sudo touch "${ZOO_LOG_DIR}/zookeeper.out"
sudo chmod -R a+wX ${ZOO_LOG_DIR}
echo "Running Zookeeper..."
sudo -E ${ZOOKEEPER_HOME}/bin/zkServer.sh start
echo "Configuring Accumulo..."
cp ${ACCUMULO_HOME}/conf/examples/1GB/standalone/* ${ACCUMULO_HOME}/conf/
rm --force ${ACCUMULO_HOME}/conf/accumulo-site.xml
Expand Down Expand Up @@ -270,7 +272,7 @@ EOF
rya-example-box
EOF
cat > ${ACCUMULO_HOME}/conf/slaves <<EOF
cat > ${ACCUMULO_HOME}/conf/slaves <<EOF
rya-example-box
EOF
sudo mkdir --parents /data
Expand All @@ -290,80 +292,85 @@ EOF
echo 'Done!'
echo "Installing RDF4J Server"
# creating log dir rdf4j-http-server-${RDF4J_VERSION}
sudo mkdir --parents /usr/share/tomcat7/.RDF4J
sudo chown -R tomcat7:tomcat7 /usr/share/tomcat7
echo "Installing RDF4J Server"
# creating log dir rdf4j-http-server-${RDF4J_VERSION}
sudo mkdir --parents /usr/share/tomcat7/.RDF4J
sudo chown -R tomcat7:tomcat7 /usr/share/tomcat7
sudo ln --force -s /usr/share/tomcat7/.RDF4J/Server/logs /var/log/tomcat7/rdf4j-server
rdf4jwar=/var/lib/tomcat7/webapps/rdf4j-server.war
if [[ ! -s $rdf4jwar ]] ; then
echo "Downloading RDF4J Server"
download --output $rdf4jwar ${mavenRepoUrl}org/eclipse/rdf4j/rdf4j-http-server/${RDF4J_VERSION}/rdf4j-http-server-${RDF4J_VERSION}.war || exit 110
fi
echo "RDF4J http server deployed at http://rya-example-box:8080/rdf4j-server"
echo "Installing RDF4J Workbench"
workbench=/var/lib/tomcat7/webapps/rdf4j-workbench.war
if [[ ! -s $workbench ]] ; then
echo "Downloading RDF4J Workbench"
download --output $workbench ${mavenRepoUrl}org/eclipse/rdf4j/rdf4j-http-workbench/${RDF4J_VERSION}/rdf4j-http-workbench-${RDF4J_VERSION}.war || exit 111
fi
echo "RDF4J workbench deployed at http://rya-example-box:8080/rdf4j-workbench"
echo "Installing Rya"
ryaIndexing=rya.indexing.example-${RYA_EXAMPLE_VERSION}-distribution
if [[ ! -s ${ryaIndexing}.zip ]] ; then
# Eventually it'll be on maven...
echo "Downloading ${ryaIndexing}.zip quietly, this will take some minutes with no output..."
download --output ${ryaIndexing}.zip ${mavenRepoUrl}org/apache/rya/rya.indexing.example/${RYA_EXAMPLE_VERSION}/${ryaIndexing}.zip || exit 112
fi
sudo mkdir --parents ${ryaIndexing}
sudo unzip -q -o ${ryaIndexing}.zip -d ${ryaIndexing}
# before continueing, wait for tomcat to deploy wars:
rdf4jwar=/var/lib/tomcat7/webapps/rdf4j-server.war
if [[ ! -s $rdf4jwar ]] ; then
echo "Downloading RDF4J Server"
download --output $rdf4jwar ${mavenRepoUrl}org/eclipse/rdf4j/rdf4j-http-server/${RDF4J_VERSION}/rdf4j-http-server-${RDF4J_VERSION}.war || exit 110
fi
echo "RDF4J http server deployed at http://rya-example-box:8080/rdf4j-server"
echo "Installing RDF4J Workbench"
workbench=/var/lib/tomcat7/webapps/rdf4j-workbench.war
if [[ ! -s $workbench ]] ; then
echo "Downloading RDF4J Workbench"
download --output $workbench ${mavenRepoUrl}org/eclipse/rdf4j/rdf4j-http-workbench/${RDF4J_VERSION}/rdf4j-http-workbench-${RDF4J_VERSION}.war || exit 111
fi
echo "RDF4J workbench deployed at http://rya-example-box:8080/rdf4j-workbench"
echo "Installing Rya"
ryaIndexing=rya.indexing.example-${RYA_EXAMPLE_VERSION}-distribution
if [[ ! -s ${ryaIndexing}.zip ]] ; then
echo "Downloading ${ryaIndexing}.zip quietly, this will take some minutes with no output..."
download --output ${ryaIndexing}.zip https://www.dropbox.com/s/gruhp3a848k8ryu/${ryaIndexing}.zip?raw=1 || exit 112
# TODO: Eventually this version of the Rya distribution will be on maven...and then we can use the following instead
#download --output ${ryaIndexing}.zip ${mavenRepoUrl}org/apache/rya/rya.indexing.example/${RYA_EXAMPLE_VERSION}/${ryaIndexing}.zip || exit 112
fi
sudo mkdir --parents ${ryaIndexing}
sudo unzip -q -o ${ryaIndexing}.zip -d ${ryaIndexing}
# before continuing, wait for tomcat to deploy wars:
waitForDeploy /var/lib/tomcat7/webapps/rdf4j-workbench/WEB-INF/lib/
waitForDeploy /var/lib/tomcat7/webapps/rdf4j-server/WEB-INF/lib/
# soft linking the files doesn't seem to work in tomcat, so we copy them instead :(
sudo cp ${ryaIndexing}/dist/lib/* /var/lib/tomcat7/webapps/rdf4j-workbench/WEB-INF/lib/ || exit 113
sudo cp ${ryaIndexing}/dist/lib/* /var/lib/tomcat7/webapps/rdf4j-server/WEB-INF/lib/ || exit 114
# These are older libs that breaks tomcat 7
sudo rm --force /var/lib/tomcat7/webapps/rdf4j-workbench/WEB-INF/lib/servlet-api-2.5.jar
sudo rm --force /var/lib/tomcat7/webapps/rdf4j-workbench/WEB-INF/lib/jsp-api-2.1.jar
sudo rm --force /var/lib/tomcat7/webapps/rdf4j-server/WEB-INF/lib/servlet-api-2.5.jar
sudo rm --force /var/lib/tomcat7/webapps/rdf4j-server/WEB-INF/lib/jsp-api-2.1.jar
sudo chown -R tomcat7:tomcat7 /var/lib/tomcat7/webapps/rdf4j-workbench/WEB-INF/lib/
sudo chown -R tomcat7:tomcat7 /var/lib/tomcat7/webapps/rdf4j-server/WEB-INF/lib/
echo "Downloading and installing new templates for RDF4J WorkBench"
ryaVagrant=rya.vagrant.example-${RYA_EXAMPLE_VERSION}
if [[ ! -s ${ryaVagrant}.jar ]] ; then
echo "Downloading ${ryaVagrant}.jar"
download --output ${ryaVagrant}.jar ${mavenRepoUrl}org/apache/rya/rya.vagrant.example/${RYA_EXAMPLE_VERSION}/${ryaVagrant}.jar || exit 120
fi
sudo mkdir --parents ${ryaVagrant}
sudo unzip -q -o ${ryaVagrant}.jar -d ${ryaVagrant}
sudo cp ${ryaVagrant}/*.xsl /var/lib/tomcat7/webapps/rdf4j-workbench/transformations/
# soft linking the files doesn't seem to work in tomcat, so we copy them instead :(
sudo cp ${ryaIndexing}/dist/lib/* /var/lib/tomcat7/webapps/rdf4j-workbench/WEB-INF/lib/ || exit 113
sudo cp ${ryaIndexing}/dist/lib/* /var/lib/tomcat7/webapps/rdf4j-server/WEB-INF/lib/ || exit 114
# These are older libs that breaks tomcat 7
sudo rm --force /var/lib/tomcat7/webapps/rdf4j-workbench/WEB-INF/lib/servlet-api-2.5.jar
sudo rm --force /var/lib/tomcat7/webapps/rdf4j-workbench/WEB-INF/lib/jsp-api-2.1.jar
sudo rm --force /var/lib/tomcat7/webapps/rdf4j-server/WEB-INF/lib/servlet-api-2.5.jar
sudo rm --force /var/lib/tomcat7/webapps/rdf4j-server/WEB-INF/lib/jsp-api-2.1.jar
sudo chown -R tomcat7:tomcat7 /var/lib/tomcat7/webapps/rdf4j-workbench/WEB-INF/lib/
sudo chown -R tomcat7:tomcat7 /var/lib/tomcat7/webapps/rdf4j-server/WEB-INF/lib/
echo "Downloading and installing new templates for RDF4J WorkBench"
ryaVagrant=rya.vagrant.example-${RYA_EXAMPLE_VERSION}
if [[ ! -s ${ryaVagrant}.jar ]] ; then
echo "Downloading ${ryaVagrant}.jar"
download --output ${ryaVagrant}.jar https://www.dropbox.com/s/p07u42et9rui3x7/${ryaVagrant}.jar?raw=1 || exit 120
# TODO: Eventually this version of the Rya distribution will be on maven...and then we can use the following instead
#download --output ${ryaVagrant}.jar ${mavenRepoUrl}org/apache/rya/rya.vagrant.example/${RYA_EXAMPLE_VERSION}/${ryaVagrant}.jar || exit 120
fi
sudo mkdir --parents ${ryaVagrant}
sudo unzip -q -o ${ryaVagrant}.jar -d ${ryaVagrant}
sudo cp ${ryaVagrant}/*.xsl /var/lib/tomcat7/webapps/rdf4j-workbench/transformations/
sudo chown tomcat7:tomcat7 /var/lib/tomcat7/webapps/rdf4j-workbench/transformations/*
echo "Deploying Rya Web"
ryaWar=web.rya-${RYA_EXAMPLE_VERSION}.war
if [[ ! -s ${ryaWar} ]] ; then
echo "Downloading ${ryaWar}"
download ${mavenRepoUrl}org/apache/rya/web.rya/${RYA_EXAMPLE_VERSION}/${ryaWar} --output ${ryaWar} || exit 121
fi
sudo cp ${ryaWar} /var/lib/tomcat7/webapps/web.rya.war
# Wait for the war to deploy
echo "Deploying Rya Web"
ryaWar=web.rya-${RYA_EXAMPLE_VERSION}.war
if [[ ! -s ${ryaWar} ]] ; then
echo "Downloading ${ryaWar}"
download https://www.dropbox.com/s/jrmgfey8ch1vrd6/${ryaWar}?raw=1 --output ${ryaWar} || exit 121
# TODO: Eventually this version of the Rya distribution will be on maven...and then we can use the following instead
#download ${mavenRepoUrl}org/apache/rya/web.rya/${RYA_EXAMPLE_VERSION}/${ryaWar} --output ${ryaWar} || exit 121
fi
sudo cp ${ryaWar} /var/lib/tomcat7/webapps/web.rya.war
# Wait for the war to deploy
waitForDeploy /var/lib/tomcat7/webapps/web.rya/WEB-INF/classes/
# These are older libs that breaks tomcat 7
sudo rm --force /var/lib/tomcat7/webapps/web.rya/WEB-INF/lib/servlet-api-2.5*.jar
sudo rm --force /var/lib/tomcat7/webapps/web.rya/WEB-INF/lib/jsp-api-2.1.jar
echo "Modify Rya Web Config"
cat > /var/lib/tomcat7/webapps/web.rya/WEB-INF/classes/environment.properties <<EOF
# These are older libs that breaks tomcat 7
sudo rm --force /var/lib/tomcat7/webapps/web.rya/WEB-INF/lib/servlet-api-2.5*.jar
sudo rm --force /var/lib/tomcat7/webapps/web.rya/WEB-INF/lib/jsp-api-2.1.jar
echo "Modify Rya Web Config"
cat > /var/lib/tomcat7/webapps/web.rya/WEB-INF/classes/environment.properties <<EOF
instance.name=dev
instance.zk=localhost:2181
instance.username=root
Expand All @@ -372,12 +379,12 @@ rya.tableprefix=rya_
rya.displayqueryplan=true
EOF
echo "Rya web deployed at http://rya-example-box:8080/web.rya/sparqlQuery.jsp"
# restart tomcat
sudo -E service tomcat7 restart
echo "Finished and ready to use!"
echo "Rya web deployed at http://rya-example-box:8080/web.rya/sparqlQuery.jsp"
# restart tomcat
sudo -E service tomcat7 restart
echo "Finished and ready to use!"
echo "You can re-apply these settings without losing data by running the command 'vagrant provision'"
SHELL

Expand Down
12 changes: 6 additions & 6 deletions extras/vagrantExample/src/main/vagrant/readme.md
View file
Expand Up @@ -31,15 +31,15 @@ The Rya Example Box VM requires Vagrant to install, configure, and run the VM.

The Rya Example Box was developed using Vagrant 1.7.4 and Oracle Virtual Box 5.0.6.

If a user would rather not use Vagrant, expert users should be able to read through the [Rya-Example-Box Vagrantfile][pathToVF] and understand the operations that the Vagrantfile is performing to configure the VM for Rya. These operations should be applicable to other Linux machines and VMs.
If a user would rather not use Vagrant, expert users should be able to read through the [Rya-Example-Box Vagrantfile] and understand the operations that the Vagrantfile is performing to configure the VM for Rya. These operations should be applicable to other Linux machines and VMs.

### Starting up the VM

Once Vagrant is installed, starting up the Rya-Example-Box is fairly straightforward.

1. Create a custom directory for the Rya Example Box (e.g., for windows users `c:\users\<user>\ryavagrant` or for Mac/Linux users `/home/<user>/ryavagrant`)

1. Download the [Rya-Example-Box Vagrantfile][pathToVF] to the custom directory. Note that it must be named `Vagrantfile` with no extension.
1. Download the [Rya-Example-Box Vagrantfile] to the custom directory. Note that it must be named `Vagrantfile` with no extension.

1. Open a DOS prompt (Windows) or Terminal (Mac/Linux), change to the custom directory, and issue the command `vagrant up`. Note that it may take up to 30 minutes to download, install, and configure Rya and all of the components.

Expand Down Expand Up @@ -187,7 +187,7 @@ From within the VM, run:

The first step to using Rya via the RDF4J Workbench is to create a repository using the Rya Accumulo Store connector.

1. Open your browser to the [RDF4j Workbench](http://rya-example-box:8080/rdf4j-workbench)
1. Open your browser to the [RDF4J Workbench](http://rya-example-box:8080/rdf4j-workbench)
2. Click on `New Repository`
3. Choose "Type" of `Rya Accumulo Store`, a Repository "ID" (e.g., `RyaAccumulo`), and a Repository "Title" (e.g., `Rya Accumulo`). Click on `Next` when complete.
4. Enter the Rya Accumulo Store connection parameters. The default parameters will connect to the Rya Example Box Acccumulo deployment (i.e., Accumulo User: `root`, Accumulo Password: `root`, Accumulo Instance: `dev`, Zookeepers: `localhost`, is Mock?: `false`). Click on `Create` when complete.
Expand Down Expand Up @@ -222,7 +222,7 @@ Once we've created a Rya repository and uploaded data, we can query Rya via the

### Using the RDF4J REST Service

More information about the RDF4J REST HTTP Protocol is availible in the [RDF4J 2.3.1 Docs]
More information about the RDF4J REST HTTP Protocol is availible in the [RDF4J 2.3.1 Docs]
(http://docs.rdf4j.org/rest-api/)

### Using Rya Java Client
Expand Down Expand Up @@ -253,6 +253,6 @@ TODO
TODO

### Resizing the VMs Disk space
Instructions for resizing a Vagrant/VirtualBox image can be found [here] (http://www.midwesternmac.com/blogs/jeff-geerling/resizing-virtualbox-disk-image)
Instructions for resizing a Vagrant/VirtualBox image can be found [here](http://www.midwesternmac.com/blogs/jeff-geerling/resizing-virtualbox-disk-image)

[pathToVF]: Vagrantfile
[Rya-Example-Box Vagrantfile]: Vagrantfile

0 comments on commit 3c3ab0d

Please sign in to comment.