From 193a3f2df2d9be060e8852fced0cd878a7f702f6 Mon Sep 17 00:00:00 2001 From: Alex Heneveld Date: Sat, 28 Jul 2018 01:08:52 +0100 Subject: [PATCH] scripts to help automate license generation for new angular js ui and tidy of generation routines, update to README --- dist/licensing/README.md | 8 +- ...te-nodejs-license-dependencies.format.json | 10 ++ .../compute-nodejs-license-dependencies.sh | 73 +++++++++ dist/licensing/generate-all.sh | 13 +- dist/licensing/generate-license-and-notice.sh | 12 +- dist/licensing/generate-one-for-testing.sh | 146 ++++++++++++++++++ ...nse-metadata-manual-1-js-pre-angular.yaml} | 0 .../license-metadata-manual-2-angular.yaml | 30 ++++ .../license-text/Font Awesome License | 34 ++++ dist/licensing/parts/license-deps | 6 +- dist/licensing/parts/yaml-asf-header | 18 +++ .../update-brooklyn-license-metadata.sh | 29 ++++ 12 files changed, 366 insertions(+), 13 deletions(-) create mode 100644 dist/licensing/compute-nodejs-license-dependencies.format.json create mode 100755 dist/licensing/compute-nodejs-license-dependencies.sh create mode 100755 dist/licensing/generate-one-for-testing.sh rename dist/licensing/{license-metadata-manual-js.yaml => license-metadata-manual-1-js-pre-angular.yaml} (100%) create mode 100644 dist/licensing/license-metadata-manual-2-angular.yaml create mode 100644 dist/licensing/license-text/Font Awesome License create mode 100644 dist/licensing/parts/yaml-asf-header create mode 100755 dist/licensing/update-brooklyn-license-metadata.sh diff --git a/dist/licensing/README.md b/dist/licensing/README.md index 30116433e0..1f5323be6b 100644 --- a/dist/licensing/README.md +++ b/dist/licensing/README.md @@ -15,9 +15,11 @@ Then in the `brooklyn-dist/dist/licensing` folder execute: This will generate updated LICENSE and NOTICE files everywhere that is needed. Compare the differences across all projects just to be sure and commit them. -* Check that the `license-inclusions-*` files in the various projects are up-to-date. - This may need manual verification for Go and JS contents (see glide.yaml for Go and package.json for JS). - +For non-Java project changes there are extra steps. Go, JS, jpegs, fonts, etc may require +special attention. Some automation is available (particularly for modern JS code) in: + + ./update-brooklyn-license-metadata.sh + # Detailed Usage diff --git a/dist/licensing/compute-nodejs-license-dependencies.format.json b/dist/licensing/compute-nodejs-license-dependencies.format.json new file mode 100644 index 0000000000..45c4d48262 --- /dev/null +++ b/dist/licensing/compute-nodejs-license-dependencies.format.json @@ -0,0 +1,10 @@ +{ + "name": "", + "version": "", + "description": "", + "repository": "", + "homepage": "", + "licenses": "", + "copyright": "", + "licenseText": "none" +} diff --git a/dist/licensing/compute-nodejs-license-dependencies.sh b/dist/licensing/compute-nodejs-license-dependencies.sh new file mode 100755 index 0000000000..9518e722e3 --- /dev/null +++ b/dist/licensing/compute-nodejs-license-dependencies.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# requires: +# node js license-checker to generate the list of deps of this project in the format used by brooklyn +# (https://github.com/davglass/license-checker) +# jq to process json (https://stedolan.github.io/jq/) +# node js json2yaml to output yaml for merging (https://www.npmjs.com/package/json2yaml) + +if [ "$1" == "--help" ] ; then + echo Computes NodeJS dependencies and outputs metadata files in `config/license-` + echo Works in current directory, or takes target directory as single argument. + exit 0 +fi + +EXCLUDE_DEPS=brooklyn + +BASE_DIR=$(pushd $(dirname $0) > /dev/null ; pwd -P ; popd > /dev/null) + +if [ ! -z "$1" ] ; then pushd $1 > /dev/null ; fi +OUTDIR=. + +if [ -d $OUTDIR/config ] ; then OUTDIR=$OUTDIR/config ; fi +TEMPFILE=$OUTDIR/license-dependencies.json.tmp + +license-checker --production --json --customPath $BASE_DIR/compute-nodejs-license-dependencies.format.json > $TEMPFILE + +cp $BASE_DIR/parts/yaml-asf-header $OUTDIR/license-inclusions-binary-autogenerated-js-license-checker.yaml +cat $TEMPFILE | jq 'keys | .[]' | awk '{ print "- id: "$0 }' | \ + grep -v "id: \"${EXCLUDE_DEPS}-" >> $OUTDIR/license-inclusions-binary-autogenerated-js-license-checker.yaml + +# a few things to note: +# * we exclude keys with null/empty values in jq +# * if homepage and repo are the same, or homepage is autogen with #readme appended, we drop homepage +# ("unique" changes the order, which we don't want) +# * json2yaml prepends --- to the file and " " on every line, which we don't want +cp $BASE_DIR/parts/yaml-asf-header $OUTDIR/license-metadata-autogenerated-js.yaml +cat $TEMPFILE | jq 'to_entries | + map({ id: .key } + + .value | { + id, name: (.name + " (NodeJS module)"), version, description, + url: ([ (select(.homepage != .repository+"#readme" and .homepage != .repository) | .homepage), .repository ] + | map(select(length > 0))), + organization: (. | { name: .publisher, email: .email, url: .url } | with_entries(select((.value | length) > 0)) ), + license: .licenses, + copyright_by: .copyright, license_text: .licenseText } + | with_entries(select((.value | length) > 0)) )' \ + | json2yaml | grep -v ^--- | sed 's/^ //' \ + >> $OUTDIR/license-metadata-autogenerated-js.yaml + +rm $TEMPFILE + +echo Reported $(wc $OUTDIR/license-inclusions-binary-autogenerated-js-license-checker.yaml | awk '{print $1}') JS dependencies for $(basename $(pwd)) + +if [ ! -z "$1" ] ; then popd > /dev/null ; fi + diff --git a/dist/licensing/generate-all.sh b/dist/licensing/generate-all.sh index 67221298c1..b8566ad698 100755 --- a/dist/licensing/generate-all.sh +++ b/dist/licensing/generate-all.sh @@ -114,7 +114,7 @@ make_for() { } -# build all the projects +# build licenses for all the projects # include deps in files pulled in to Go CLI binary builds make_for $ROOT_DIR/brooklyn-client/cli/ release/license/files binary-primary @@ -123,8 +123,14 @@ make_for $ROOT_DIR/brooklyn-client/cli/ . binary-additional # Server CLI has embedded JS; gets custom files in sub-project root, also included in JAR make_for $ROOT_DIR/brooklyn-server/server-cli/ . binary-additional -# UI gets files at root, also included in WAR +# UI gets files at root make_for $ROOT_DIR/brooklyn-ui/ . binary-additional +# for UI also do for each standalone module +for x in $(ls $ROOT_DIR/brooklyn-ui/ui-modules/*/package.json) ; do + make_for ${x%package.json} . binary-additional + # and in modules which make a WAR/JAR files we embed binaries + if [ -d ${x%package.json}/src/main/webapp ] ; then make_for ${x%package.json} src/main/webapp/WEB-INF/classes/META-INF/ binary-primary ; fi +done # main projects have their binaries included at root make_for $ROOT_DIR/brooklyn-server/ . binary-additional @@ -141,4 +147,5 @@ make_for $ROOT_DIR/brooklyn-dist/dist src/main/license/files/ binary-primary $RO cp $OUT/{NOTICE,LICENSE} $PROJ/../karaf/apache-brooklyn/src/main/resources/ # finally in root project list everything -make_for $ROOT_DIR/brooklyn-dist/dist ../.. binary-additional $ROOT_DIR \ No newline at end of file +make_for $ROOT_DIR/brooklyn-dist/dist ../.. binary-additional $ROOT_DIR + diff --git a/dist/licensing/generate-license-and-notice.sh b/dist/licensing/generate-license-and-notice.sh index 0e6ee57461..120cf27be9 100755 --- a/dist/licensing/generate-license-and-notice.sh +++ b/dist/licensing/generate-license-and-notice.sh @@ -131,7 +131,8 @@ TEMP_MVN_OUT=`pwd -P`/temp.license-maven-output.log echo > $TEMP_METADATA_FILE if [ ! -z "$LIBRARIES" ] ; then echo Using metadata libraries $(find "${LIBRARIES[@]}" -name "license-metadata-*") - for x in $(find "${LIBRARIES[@]}" -name "license-metadata-*") ; do + # sort by filename first, then by path, with later ones alpha being the ones that are ultimately used + for x in $(find "${LIBRARIES[@]}" -name "license-metadata-*" | sed 's/\(.*\/\)\(.*\)/\2 --- \1\2/' | sort | sed 's/.* --- //') ; do cat $x >> $TEMP_METADATA_FILE done fi @@ -179,19 +180,22 @@ cat ${TEMP_LICENSES_}1 | while read x ; do echo $x ; done | sort | uniq > ${TEMP MISSING=() if [ -s ${TEMP_LICENSES_}2 ] ; then - echo Adding ${LICENSES[$I]} to $LICENSE_FILE and `cat ${TEMP_LICENSES_}2 | wc -l` licenses + echo Adding ${LICENSES[$I]} to $LICENSE_FILE and `cat ${TEMP_LICENSES_}2 | wc -l` licenses:`cat ${TEMP_LICENSES_}2 | sed 's/^/ /' | paste -sd ';' -` cat ${LICENSES[${#LICENSES[@]}-1]} >> $LICENSE_FILE - + LICENSE_TEXT_PATHS=$(find ${LIBRARIES[@]} -name license-text) + while read x ; do echo " "$x": |" >> $LICENSE_FILE unset FOUND - for lp in $(find ${LIBRARIES[@]} -name license-text) ; do + for lp in ${LICENSE_TEXT_PATHS} ; do if [ -f "$lp/$x" ] ; then + if [ -z "$FOUND" ]; then cat "$lp/$x" | sed "s/^/ /" >> $LICENSE_FILE echo "" >> $LICENSE_FILE FOUND=true + fi fi done if [ -z "$FOUND" ]; then diff --git a/dist/licensing/generate-one-for-testing.sh b/dist/licensing/generate-one-for-testing.sh new file mode 100755 index 0000000000..4b20c6928c --- /dev/null +++ b/dist/licensing/generate-one-for-testing.sh @@ -0,0 +1,146 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e + +usage() { + cat >&2 << EOF + +Usage: generate-all.sh + +Execute generate-license-and-notice.sh to generate LICENSE and NOTICE files for all Brooklyn projects. + +EOF +} + +while [ ! -z "$*" ] ; do + + if [ "$1" == "--help" ]; then usage ; exit 0; fi + + usage + echo Unexpected argument: $1 + exit 1 + +done + + +REF_DIR=$(pushd $(dirname $0) > /dev/null ; pwd -P ; popd > /dev/null) +PARTS_DIR=$REF_DIR/parts +ROOT_DIR=$REF_DIR/../../.. +MVN_OUTFILE=$REF_DIR/notices.autogenerated + +prefix_and_join_array() { + PREFIX=$2 + JOIN_BEFORE_PREFIX=$1 + JOIN_AFTER_PREFIX=$3 + echo -n ${PREFIX}$4 + shift 4 + while (($# >= 1)) ; do + echo -n "${JOIN_BEFORE_PREFIX}${PREFIX}${JOIN_AFTER_PREFIX}$1" + shift + done +} + +# takes root dir in first arg, then regex expression +make_for() { + PROJ=$(cd $1 ; pwd -P) + OUT=${PROJ}/$2 + MODE=$3 + SEARCH_ROOT=$4 + if [ -z "$SEARCH_ROOT" ] ; then SEARCH_ROOT=$PROJ ; fi + + echo Generating for $PROJ mode $MODE to $2... + echo "" + + pushd $PROJ > /dev/null + + if [ "$MODE" == "binary-additional" ] ; then + + $REF_DIR/generate-license-and-notice.sh \ + -o $OUT \ + --license $PARTS_DIR/license-top \ + --license $PARTS_DIR/license-deps \ + --notice $PARTS_DIR/notice-top --notice-compute-with-flags " + -DextrasFiles=$(prefix_and_join_array "" ":" "" $(find $SEARCH_ROOT -name "license-inclusions-source-*")) + -DonlyExtras=true" \ + --notice $PARTS_DIR/notice-additional --notice-compute-with-flags " + -DextrasFiles=$(prefix_and_join_array "" ":" "" $(find $SEARCH_ROOT -name "license-inclusions-binary-*"))" \ + --libraries ${REF_DIR} ${SEARCH_ROOT} + + elif [ "$MODE" == "binary-primary" ] ; then + + $REF_DIR/generate-license-and-notice.sh \ + -o $OUT \ + --license $PARTS_DIR/license-top \ + --license $PARTS_DIR/license-deps \ + --notice $PARTS_DIR/notice-top --notice-compute-with-flags " + -DextrasFiles=$(prefix_and_join_array "" ":" "" $(find $SEARCH_ROOT -name "license-inclusions-source-*" -or -name "license-inclusions-binary-*"))" \ + --libraries ${REF_DIR} ${SEARCH_ROOT} + + elif [ "$MODE" == "binary-omitted" ] ; then + + $REF_DIR/generate-license-and-notice.sh \ + -o $OUT \ + --license $PARTS_DIR/license-top \ + --license $PARTS_DIR/license-deps \ + --notice $PARTS_DIR/notice-top --notice-compute-with-flags " + -DextrasFiles=$(prefix_and_join_array "" ":" "" $(find $SEARCH_ROOT -name "license-inclusions-source-*")) + -DonlyExtras=true" \ + --libraries ${REF_DIR} ${SEARCH_ROOT} + + else + echo FAILED - unknown mode $MODE + exit 1 + fi + echo "" + + popd > /dev/null +} + + +# build all the projects + +make_for $ROOT_DIR/brooklyn-ui/ui-modules/home . binary-additional +exit + +# include deps in files pulled in to Go CLI binary builds +make_for $ROOT_DIR/brooklyn-client/cli/ release/license/files binary-primary +make_for $ROOT_DIR/brooklyn-client/cli/ . binary-additional + +# Server CLI has embedded JS; gets custom files in sub-project root, also included in JAR +make_for $ROOT_DIR/brooklyn-server/server-cli/ . binary-additional + +# UI gets files at root, also included in WAR + +# main projects have their binaries included at root +make_for $ROOT_DIR/brooklyn-server/ . binary-additional +make_for $ROOT_DIR/brooklyn-client/ . binary-additional +make_for $ROOT_DIR/brooklyn-library/ . binary-additional +# dist is trickier, just don't mention binaries in the generated items +make_for $ROOT_DIR/brooklyn-dist/ . binary-omitted + +# brooklyn-docs skipped +# the docs don't make a build and don't include embedded code so no special license there + +# and the binary dists; dist/ project which has biggest deps set, but search in all brooklyn projects +make_for $ROOT_DIR/brooklyn-dist/dist src/main/license/files/ binary-primary $ROOT_DIR +cp $OUT/{NOTICE,LICENSE} $PROJ/../karaf/apache-brooklyn/src/main/resources/ + +# finally in root project list everything +make_for $ROOT_DIR/brooklyn-dist/dist ../.. binary-additional $ROOT_DIR diff --git a/dist/licensing/license-metadata-manual-js.yaml b/dist/licensing/license-metadata-manual-1-js-pre-angular.yaml similarity index 100% rename from dist/licensing/license-metadata-manual-js.yaml rename to dist/licensing/license-metadata-manual-1-js-pre-angular.yaml diff --git a/dist/licensing/license-metadata-manual-2-angular.yaml b/dist/licensing/license-metadata-manual-2-angular.yaml new file mode 100644 index 0000000000..f2b199782e --- /dev/null +++ b/dist/licensing/license-metadata-manual-2-angular.yaml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +# overrides file for org.heneveld.license-audit-maven-plugin +# expands/corrects detail from autogenerated node js metadata files + +- id: font-awesome@4.7.0 + license: + name: Font Awesome License + url: https://github.com/FortAwesome/Font-Awesome/blob/master/LICENSE.txt + +- id: identicon.js@1.3.0 + license: BSD-2-Clause diff --git a/dist/licensing/license-text/Font Awesome License b/dist/licensing/license-text/Font Awesome License new file mode 100644 index 0000000000..28c1c4bc73 --- /dev/null +++ b/dist/licensing/license-text/Font Awesome License @@ -0,0 +1,34 @@ +Font Awesome Free License +------------------------- + +Font Awesome Free is free, open source, and GPL friendly. You can use it for +commercial projects, open source projects, or really almost whatever you want. +Full Font Awesome Free license: https://fontawesome.com/license. + +# Icons: CC BY 4.0 License (https://creativecommons.org/licenses/by/4.0/) +In the Font Awesome Free download, the CC BY 4.0 license applies to all icons +packaged as SVG and JS file types. + +# Fonts: SIL OFL 1.1 License (https://scripts.sil.org/OFL) +In the Font Awesome Free download, the SIL OLF license applies to all icons +packaged as web and desktop font files. + +# Code: MIT License (https://opensource.org/licenses/MIT) +In the Font Awesome Free download, the MIT license applies to all non-font and +non-icon files. + +# Attribution +Attribution is required by MIT, SIL OLF, and CC BY licenses. Downloaded Font +Awesome Free files already contain embedded comments with sufficient +attribution, so you shouldn't need to do anything additional when using these +files normally. + +We've kept attribution comments terse, so we ask that you do not actively work +to remove them from files, especially code. They're a great way for folks to +learn about Font Awesome. + +# Brand Icons +All brand icons are trademarks of their respective owners. The use of these +trademarks does not indicate endorsement of the trademark holder by Font +Awesome, nor vice versa. **Please do not use brand logos for any purpose except +to represent the company, product, or service to which they refer.** diff --git a/dist/licensing/parts/license-deps b/dist/licensing/parts/license-deps index 483b985083..c534e67353 100644 --- a/dist/licensing/parts/license-deps +++ b/dist/licensing/parts/license-deps @@ -1,8 +1,8 @@ Dependency licenses: - # This software includes dependencies released under other licenses. - # These other licenses are compatible with the Apache License above. + # This software includes dependencies released under their licenses. + # These licenses are compatible with the Apache License above. # Details of these dependencies can be found in the accompanying NOTICE file. - # These other licenses are included below with their full text. + # These licenses are included below with their full text. diff --git a/dist/licensing/parts/yaml-asf-header b/dist/licensing/parts/yaml-asf-header new file mode 100644 index 0000000000..30097ef048 --- /dev/null +++ b/dist/licensing/parts/yaml-asf-header @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# diff --git a/dist/licensing/update-brooklyn-license-metadata.sh b/dist/licensing/update-brooklyn-license-metadata.sh new file mode 100755 index 0000000000..d795e53236 --- /dev/null +++ b/dist/licensing/update-brooklyn-license-metadata.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +# for Go code, see the glide.yaml in those projects; you'll have to manually update + + +# for new JS UI we have good automation: + +for x in $(ls ../../../brooklyn-ui/ui-modules/*/package.json) ; do + ./compute-nodejs-license-dependencies.sh ${x%package.json} +done