Merge pull request #4421 from IQSS/develop

v4.8.5

.gitignore

@@ -33,3 +33,11 @@ scripts/api/setup-all.sh*
 
 # ctags generated tag file
 tags
+
+# dependencies I'm not sure we're allowed to redistribute / have in version control
+conf/docker-aio/dv/deps/
+
+# no need to check aoi installer zip into vc
+conf/docker-aio/dv/install/dvinstall.zip
+# or copy of test data
+conf/docker-aio/testdata/

Vagrantfile

@@ -8,18 +8,13 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   config.vm.define "standalone", primary: true do |standalone|
     config.vm.hostname = "standalone"
-    standalone.vm.box = "puppet-vagrant-boxes.puppetlabs.com-centos-65-x64-virtualbox-puppet.box"
+    # Uncomment this temporarily to get `vagrant destroy` to work
+    #standalone.vm.box = "puppetlabs/centos-7.2-64-puppet"
 
     operating_system = "centos"
     if ENV['OPERATING_SYSTEM'].nil?
-      puts "OPERATING_SYSTEM environment variable not specified. Using #{operating_system} by default.\nTo specify it in bash: export OPERATING_SYSTEM=debian"
-      config.vm.box_url = "http://puppet-vagrant-boxes.puppetlabs.com/centos-65-x64-virtualbox-puppet.box"
-      config.vm.box = "puppet-vagrant-boxes.puppetlabs.com-centos-65-x64-virtualbox-puppet.box"
-    elsif ENV['OPERATING_SYSTEM'] == 'centos7'
-      puts "WARNING: CentOS 7 specified. Newer than what the dev team tests on."
-      config.vm.box_url = "https://atlas.hashicorp.com/puppetlabs/boxes/centos-7.2-64-puppet/versions/1.0.1/providers/virtualbox.box"
-      config.vm.box = "puppetlabs-centos-7.2-64-puppet-1.0.1-virtualbox.box"
-      standalone.vm.box = "puppetlabs-centos-7.2-64-puppet-1.0.1-virtualbox.box"
+      config.vm.box = "puppetlabs/centos-7.2-64-puppet"
+      config.vm.box_version = '1.0.1'
     elsif ENV['OPERATING_SYSTEM'] == 'debian'
       puts "WARNING: Debian specified. Here be dragons! https://github.com/IQSS/dataverse/issues/1059"
       config.vm.box_url = "http://puppet-vagrant-boxes.puppetlabs.com/debian-73-x64-virtualbox-puppet.box"

conf/docker-aio/0prep_deps.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+if [ ! -d dv/deps ]; then
+	mkdir -p dv/deps
+fi
+wdir=`pwd`
+if [ ! -e dv/deps/glassfish4dv.tgz ]; then
+	echo "glassfish dependency prep"
+	mkdir -p /tmp/dv-prep/gf
+	cd /tmp/dv-prep/gf
+	wget http://download.java.net/glassfish/4.1/release/glassfish-4.1.zip
+	wget http://search.maven.org/remotecontent?filepath=org/jboss/weld/weld-osgi-bundle/2.2.10.Final/weld-osgi-bundle-2.2.10.Final-glassfish4.jar -O weld-osgi-bundle-2.2.10.Final-glassfish4.jar
+	unzip glassfish-4.1.zip
+	rm glassfish4/glassfish/modules/weld-osgi-bundle.jar
+	mv weld-osgi-bundle-2.2.10.Final-glassfish4.jar glassfish4/glassfish/modules
+	tar zcf $wdir/dv/deps/glassfish4dv.tgz glassfish4
+	cd $wdir
+	# assuming that folks usually have /tmp auto-clean as needed
+fi
+
+if [ ! -e dv/deps/solr-4.6.0dv.tgz ]; then
+	echo "solr dependency prep"
+	# schema changes *should* be the only ones...
+	cd dv/deps/
+	wget https://archive.apache.org/dist/lucene/solr/4.6.0/solr-4.6.0.tgz -O solr-4.6.0dv.tgz
+	cd ../../
+fi
+

conf/docker-aio/1prep.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# move things necessary for integration tests into build context.
+# this was based off the phoenix deployment; and is likely uglier and bulkier than necessary in a perfect world
+
+mkdir -p testdata/doc/sphinx-guides/source/_static/util/
+cp ../solr/4.6.0/schema.xml testdata/
+cp ../jhove/jhove.conf testdata/
+cd ../../
+cp -r scripts conf/docker-aio/testdata/
+cp doc/sphinx-guides/source/_static/util/pg8-createsequence-prep.sql conf/docker-aio/testdata/doc/sphinx-guides/source/_static/util/
+cp doc/sphinx-guides/source/_static/util/createsequence.sql conf/docker-aio/testdata/doc/sphinx-guides/source/_static/util/
+
+# not using dvinstall.zip for setupIT.bash; but still used in install.bash for normal ops
+mvn clean
+./scripts/database/homebrew/custom-build-number
+mvn package
+cd scripts/installer
+make clean
+make
+cp dvinstall.zip ../../conf/docker-aio/dv/install/
+

conf/docker-aio/c7.dockerfile

@@ -0,0 +1,40 @@
+FROM centos:7
+# OS dependencies
+RUN yum install -y java-1.8.0-openjdk-headless postgresql-server sudo epel-release unzip perl curl
+RUN yum install -y jq
+
+# copy and unpack dependencies (solr, glassfish)
+COPY dv /tmp/dv
+COPY testdata/schema.xml /tmp/dv
+RUN cd /opt ; tar zxf /tmp/dv/deps/solr-4.6.0dv.tgz 
+RUN cd /opt ; tar zxf /tmp/dv/deps/glassfish4dv.tgz
+
+RUN sudo -u postgres /usr/bin/initdb -D /var/lib/pgsql/data
+#RUN sudo -u postgres createuser dvnapp
+
+# copy configuration related files
+RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/data/ ; cp /tmp/dv/schema.xml /opt/solr-4.6.0/example/solr/collection1/conf/schema.xml
+
+# skipping glassfish user and solr user (run both as root)
+
+#solr port
+EXPOSE 8983
+
+# postgres port
+EXPOSE 5432
+
+# glassfish port
+EXPOSE 8080
+
+RUN mkdir /opt/dv
+
+# yeah - still not happy if glassfish isn't in /usr/local :<
+RUN ln -s /opt/glassfish4 /usr/local/glassfish4
+COPY dv/install/ /opt/dv/
+COPY install.bash /opt/dv/
+COPY entrypoint.bash /opt/dv/
+COPY testdata /opt/dv/testdata
+COPY testscripts/* /opt/dv/testdata/
+COPY setupIT.bash /opt/dv
+WORKDIR /opt/dv
+CMD ["/opt/dv/entrypoint.bash"]

conf/docker-aio/default.config

@@ -0,0 +1,16 @@
+HOST_DNS_ADDRESS	localhost
+GLASSFISH_DIRECTORY	/opt/glassfish4
+ADMIN_EMAIL	 
+MAIL_SERVER	mail.hmdc.harvard.edu
+POSTGRES_ADMIN_PASSWORD	secret
+POSTGRES_SERVER	db
+POSTGRES_PORT	5432
+POSTGRES_DATABASE	dvndb
+POSTGRES_USER	dvnapp
+POSTGRES_PASSWORD	secret
+SOLR_LOCATION	idx
+TWORAVENS_LOCATION	NOT INSTALLED
+RSERVE_HOST	localhost
+RSERVE_PORT	6311
+RSERVE_USER	rserve
+RSERVE_PASSWORD	rserve

conf/docker-aio/dv/install/default.config

@@ -0,0 +1,16 @@
+HOST_DNS_ADDRESS	localhost
+GLASSFISH_DIRECTORY	/opt/glassfish4
+ADMIN_EMAIL	 
+MAIL_SERVER	mail.hmdc.harvard.edu
+POSTGRES_ADMIN_PASSWORD	secret
+POSTGRES_SERVER	db
+POSTGRES_PORT	5432
+POSTGRES_DATABASE	dvndb
+POSTGRES_USER	dvnapp
+POSTGRES_PASSWORD	secret
+SOLR_LOCATION	idx
+TWORAVENS_LOCATION	NOT INSTALLED
+RSERVE_HOST	localhost
+RSERVE_PORT	6311
+RSERVE_USER	rserve
+RSERVE_PASSWORD	rserve

conf/docker-aio/dv/pg_hba.conf

@@ -0,0 +1,91 @@
+# PostgreSQL Client Authentication Configuration File
+# ===================================================
+#
+# Refer to the "Client Authentication" section in the PostgreSQL
+# documentation for a complete description of this file.  A short
+# synopsis follows.
+#
+# This file controls: which hosts are allowed to connect, how clients
+# are authenticated, which PostgreSQL user names they can use, which
+# databases they can access.  Records take one of these forms:
+#
+# local      DATABASE  USER  METHOD  [OPTIONS]
+# host       DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
+# hostssl    DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
+# hostnossl  DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
+#
+# (The uppercase items must be replaced by actual values.)
+#
+# The first field is the connection type: "local" is a Unix-domain
+# socket, "host" is either a plain or SSL-encrypted TCP/IP socket,
+# "hostssl" is an SSL-encrypted TCP/IP socket, and "hostnossl" is a
+# plain TCP/IP socket.
+#
+# DATABASE can be "all", "sameuser", "samerole", "replication", a
+# database name, or a comma-separated list thereof. The "all"
+# keyword does not match "replication". Access to replication
+# must be enabled in a separate record (see example below).
+#
+# USER can be "all", a user name, a group name prefixed with "+", or a
+# comma-separated list thereof.  In both the DATABASE and USER fields
+# you can also write a file name prefixed with "@" to include names
+# from a separate file.
+#
+# ADDRESS specifies the set of hosts the record matches.  It can be a
+# host name, or it is made up of an IP address and a CIDR mask that is
+# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that
+# specifies the number of significant bits in the mask.  A host name
+# that starts with a dot (.) matches a suffix of the actual host name.
+# Alternatively, you can write an IP address and netmask in separate
+# columns to specify the set of hosts.  Instead of a CIDR-address, you
+# can write "samehost" to match any of the server's own IP addresses,
+# or "samenet" to match any address in any subnet that the server is
+# directly connected to.
+#
+# METHOD can be "trust", "reject", "md5", "password", "gss", "sspi",
+# "krb5", "ident", "peer", "pam", "ldap", "radius" or "cert".  Note that
+# "password" sends passwords in clear text; "md5" is preferred since
+# it sends encrypted passwords.
+#
+# OPTIONS are a set of options for the authentication in the format
+# NAME=VALUE.  The available options depend on the different
+# authentication methods -- refer to the "Client Authentication"
+# section in the documentation for a list of which options are
+# available for which authentication methods.
+#
+# Database and user names containing spaces, commas, quotes and other
+# special characters must be quoted.  Quoting one of the keywords
+# "all", "sameuser", "samerole" or "replication" makes the name lose
+# its special character, and just match a database or username with
+# that name.
+#
+# This file is read on server startup and when the postmaster receives
+# a SIGHUP signal.  If you edit the file on a running system, you have
+# to SIGHUP the postmaster for the changes to take effect.  You can
+# use "pg_ctl reload" to do that.
+
+# Put your actual configuration here
+# ----------------------------------
+#
+# If you want to allow non-local connections, you need to add more
+# "host" records.  In that case you will also need to make PostgreSQL
+# listen on a non-local interface via the listen_addresses
+# configuration parameter, or via the -i or -h command line switches.
+
+
+
+# TYPE  DATABASE        USER            ADDRESS                 METHOD
+
+# "local" is for Unix domain socket connections only
+#local   all             all                                     peer
+local   all             all                                     trust
+# IPv4 local connections:
+#host    all             all             127.0.0.1/32            trust
+host    all             all             0.0.0.0/0            trust
+# IPv6 local connections:
+host    all             all             ::1/128                 trust
+# Allow replication connections from localhost, by a user with the
+# replication privilege.
+#local   replication     postgres                                peer
+#host    replication     postgres        127.0.0.1/32            ident
+#host    replication     postgres        ::1/128                 ident

conf/docker-aio/entrypoint.bash

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+sudo -u postgres /usr/bin/postgres -D /var/lib/pgsql/data &
+cd /opt/solr-4.6.0/example/
+java -DSTOP.PORT=8079 -DSTOP.KEY=a09df7a0d -jar start.jar &
+
+cd /opt/glassfish4
+bin/asadmin start-domain
+sleep infinity
+

conf/docker-aio/install.bash

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+sudo -u postgres createuser --superuser dvnapp
+#./entrypoint.bash &
+unzip dvinstall.zip
+cd dvinstall/
+./install -admin_email=pameyer+dvinstall@crystal.harvard.edu -y -f > install.out 2> install.err
+
+echo "installer complete"
+cat install.err

conf/docker-aio/readme.txt

@@ -0,0 +1,23 @@
+first pass docker all-in-one image, intended for running integration tests against.
+
+Could be potentially usable for normal development as well.
+
+
+Initial setup (aka - do once):
+- Do surgery on glassfish4 and solr4.6.0 following guides, place results in `conf/docker-aio/dv/deps` as `glassfish4dv.tgz` and `solr-4.6.0dv.tgz` respectively. Running `conf/docker-aio/0prep_deps.sh` attempts to automate this.
+
+Per-build:
+- `cd conf/docker-aio`, and run `1prep.sh` to copy files for integration test data into docker build context; `1prep.sh` will also build the war file and installation zip file
+- build the docker image: `docker build -t dv0 -f c7.dockerfile .`
+
+- Run image: `docker run -d -p 8083:8080 --name dv dv0` (aka - forward port 8083 locally to 8080 in the container)
+- Installation (integration test): `docker exec -it dv /opt/dv/setupIT.bash`
+- Installation (non-interactive, uses `conf/docker-aio/default.config`): `docker exec -it dv /opt/dv/install.bash`
+
+- update `dataverse.siteUrl` (appears only necessary for `DatasetsIT.testPrivateUrl`): `docker exec -it dv /usr/local/glassfish4/bin/asadmin create-jvm-options "-Ddataverse.siteUrl=http\://localhost\:8083"`
+
+Run integration tests: 
+`mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT -Ddataverse.test.baseurl='http://localhost:8083'`
+
+There isn't any strict requirement on the local port (8083 in this doc), the name of the image (dv0) or container (dv), these can be changed as desired as long as they are consistent.
+

conf/docker-aio/setupIT.bash

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# do integration-test install and test data setup
+
+cd /opt/dv
+unzip dvinstall.zip
+cd /opt/dv/testdata
+./scripts/deploy/phoenix.dataverse.org/prep
+./db.sh
+./install # modified from phoenix
+/usr/local/glassfish4/glassfish/bin/asadmin deploy /opt/dv/dvinstall/dataverse.war
+./post # modified from phoenix
+

conf/docker-aio/testscripts/db.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+psql -U postgres -c "CREATE ROLE dvnapp UNENCRYPTED PASSWORD 'secret' SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN" template1
+psql -U dvnapp -c 'CREATE DATABASE "dvndb" WITH OWNER = "dvnapp"' template1

conf/docker-aio/testscripts/install

@@ -0,0 +1,21 @@
+#!/bin/sh
+export HOST_ADDRESS=localhost
+export GLASSFISH_ROOT=/usr/local/glassfish4
+export FILES_DIR=/usr/local/glassfish4/glassfish/domains/domain1/files
+export DB_NAME=dvndb
+export DB_PORT=5432
+export DB_HOST=localhost
+export DB_USER=dvnapp
+export DB_PASS=secret
+export RSERVE_HOST=localhost
+export RSERVE_PORT=6311
+export RSERVE_USER=rserve
+export RSERVE_PASS=rserve
+export SMTP_SERVER=localhost
+export MEM_HEAP_SIZE=2048
+export GLASSFISH_DOMAIN=domain1
+cd scripts/installer
+cp pgdriver/postgresql-8.4-703.jdbc4.jar $GLASSFISH_ROOT/glassfish/lib
+#cp ../../conf/jhove/jhove.conf $GLASSFISH_ROOT/glassfish/domains/$GLASSFISH_DOMAIN/config/jhove.conf
+cp /opt/dv/testdata/jhove.conf $GLASSFISH_ROOT/glassfish/domains/$GLASSFISH_DOMAIN/config/jhove.conf
+./glassfish-setup.sh

conf/docker-aio/testscripts/post

@@ -0,0 +1,15 @@
+#/bin/sh
+cd scripts/api
+./setup-all.sh --insecure | tee /tmp/setup-all.sh.out
+cd ../..
+psql -U dvnapp dvndb -f scripts/database/reference_data.sql
+psql -U dvnapp dvndb -f doc/sphinx-guides/source/_static/util/pg8-createsequence-prep.sql
+psql -U dvnapp dvndb -f doc/sphinx-guides/source/_static/util/createsequence.sql
+scripts/search/tests/publish-dataverse-root
+#git checkout scripts/api/data/dv-root.json
+scripts/search/tests/grant-authusers-add-on-root
+scripts/search/populate-users
+scripts/search/create-users
+scripts/search/tests/create-all-and-test
+scripts/search/tests/publish-spruce1-and-test
+#java -jar downloads/schemaSpy_5.0.0.jar -t pgsql -host localhost -db dvndb -u postgres -p secret -s public -dp scripts/installer/pgdriver/postgresql-9.1-902.jdbc4.jar -o /var/www/html/schemaspy/latest

doc/sphinx-guides/source/_static/admin/ipGroupAll.json

@@ -0,0 +1,14 @@
+{
+  "alias": "ipGroupAll",
+  "name": "IP group to match all IPv4 and IPv6 addresses",
+  "ranges": [
+    [
+      "0.0.0.0",
+      "255.255.255.255"
+    ],
+    [
+      "::",
+      "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"
+    ]
+  ]
+}

doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service

@@ -0,0 +1,13 @@
+[Unit]
+Description = Apache Solr
+After = syslog.target network.target remote-fs.target nss-lookup.target
+
+[Service]
+User = solr
+Type = simple
+WorkingDirectory = /usr/local/solr/example
+ExecStart = /usr/bin/java -jar -server /usr/local/solr/example/start.jar
+Restart=on-failure
+
+[Install]
+WantedBy = multi-user.target

doc/sphinx-guides/source/_static/installation/files/root/external-tools/awesomeTool.json

@@ -0,0 +1,15 @@
+{
+  "displayName": "Awesome Tool",
+  "description": "The most awesome tool.",
+  "toolUrl": "https://awesometool.com",
+  "toolParameters": {
+    "queryParameters": [
+      {
+        "fileid": "{fileId}"
+      },
+      {
+        "key": "{apiToken}"
+      }
+    ]
+  }
+}