[PIO-69] Update LICENSE.txt and NOTICE.txt for binary distribution

and this fix contains following stuffs:

- Include these files in the binary distribution
- Enable license check using sbt-license-report plugin in travis build

Closes #397

.travis.yml

@@ -108,6 +108,8 @@ env:
       PIO_SPARK_VERSION=2.1.1
       PIO_ELASTICSEARCH_VERSION=5.4.1
 
+    - BUILD_TYPE=LicenseCheck
+
 before_install:
   - unset SBT_OPTS JVM_OPTS
   - sudo rm /usr/local/bin/docker-compose

NOTICE.txt

@@ -3,3 +3,7 @@ Copyright 2016 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
+
+This product depends on third party software that falls under a variety of licenses.
+All dependencies with licenses other than Apache are specified in the LICENSE file.
+Please see LICENSE for additional copyright and licensing information.

build.sbt

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
+import com.typesafe.sbt.license.{LicenseInfo, DepModuleInfo}
 import PIOBuild._
 
 lazy val scalaSparkDepsVersion = Map(
@@ -98,6 +98,7 @@ val conf = file("conf")
 
 val commonSettings = Seq(
   autoAPIMappings := true,
+  licenseConfigurations := Set("compile"),
   unmanagedClasspath in Test += conf,
   unmanagedClasspath in Test += baseDirectory.value.getParentFile / s"storage/jdbc/target/scala-${scalaBinaryVersion.value}/classes")
 

make-distribution.sh

@@ -107,6 +107,9 @@ TARNAME="PredictionIO-$VERSION.tar.gz"
 TARDIR="PredictionIO-$VERSION"
 cp -r ${DISTDIR} ${TARDIR}
 
+cp LICENSE.txt ${TARDIR}
+cp NOTICE.txt ${TARDIR}
+
 tar zcvf ${TARNAME} ${TARDIR}
 rm -rf ${TARDIR}
 

project/plugins.sbt

@@ -13,3 +13,5 @@ resolvers += "sonatype-releases" at "https://oss.sonatype.org/content/repositori
 addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.3.5")
 
 addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.2.0-M8")
+
+addSbtPlugin("com.typesafe.sbt" % "sbt-license-report" % "1.2.0")

tests/check_libraries.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Go to PredictionIO directory
+FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
+mkdir -p ${FWDIR}/lib
+cd ${FWDIR}
+
+REPORT_DIR="${FWDIR}/test-reports"
+GATHERED_FILE="${REPORT_DIR}/licences-gathered.csv"
+FILTERED_FILE="${REPORT_DIR}/licences-filtered.csv"
+ERROR_FILE="${REPORT_DIR}/licences-errors.csv"
+
+# Extract libraries which are not described in LICENSE.txt
+echo "Check library dependencies..."
+
+# Generate license report
+sbt/sbt clean
+sbt/sbt dumpLicenseReport
+
+sbt/sbt storage/clean
+sbt/sbt storage/dumpLicenseReport
+
+# Clean up
+mkdir -p ${REPORT_DIR}
+
+rm -f ${GATHERED_FILE}
+rm -f ${FILTERED_FILE}
+rm -f ${ERROR_FILE}
+
+# Gather and filter reports
+find . -name "*-licenses.csv" -exec cat {} >> ${GATHERED_FILE} \;
+cat ${GATHERED_FILE} | sort | uniq | grep -v "Category,License,Dependency,Notes" | \
+  grep -v "Apache" | \
+  grep -v "ASL" | \
+  grep -v "org.apache" | \
+  grep -v "commons-" | \
+  grep -v "tomcat" | \
+  grep -v "org.codehaus.jettison" | \
+  grep -v "xml-apis" | \
+  grep -v "org.mortbay.jetty" | \
+  grep -v "com.google.guava" | \
+  grep -v "predictionio" > ${FILTERED_FILE}
+
+# Check undocumented
+cat ${FILTERED_FILE} | while read LINE
+do
+  LIBRARY=`echo ${LINE} | cut -d ',' -f 3`
+  grep -q "$LIBRARY" "${FWDIR}/LICENSE.txt"
+  if [ $? -ne 0 ]; then
+    echo -e "\033[0;31m[error]\033[0;39m Undocumented dependency: $LINE"
+    echo "Undocumented dependency: $LINE" >> ${ERROR_FILE}
+  fi
+done
+
+# Extract libraries which are described in LICENSE.txt but not exist actually
+echo "Check libraries described in LICENSE.txt..."
+
+cat "${FWDIR}/LICENSE.txt" | grep "#" | sed -e 's/(.*)//' | sed -e '/^#/d' | while read LINE
+do
+  grep -q "$LINE" ${GATHERED_FILE}
+  if [ $? -ne 0 ]; then
+    echo -e "\033[0;31m[error]\033[0;39m Can't find: $LINE"
+    echo "Unused dependency: $LINE" >> ${ERROR_FILE}
+  fi
+done
+
+if [ -f ${ERROR_FILE} ]; then
+  echo "Library checks failed."
+  exit 1
+else 
+  echo "Library checks passed."
+  exit 0
+fi  

tests/script.travis.sh

@@ -19,6 +19,8 @@
 if [[ $BUILD_TYPE == Unit ]]; then
   tests/run_docker.sh $METADATA_REP $EVENTDATA_REP $MODELDATA_REP \
     "/PredictionIO/tests/unit.sh"
+elif [[ $BUILD_TYPE == LicenseCheck ]]; then
+  tests/check_libraries.sh
 else
   tests/run_docker.sh $METADATA_REP $EVENTDATA_REP $MODELDATA_REP \
     "python3 /PredictionIO/tests/pio_tests/tests.py"