Skip to content
Permalink
Browse files
Binary license management system (#7998)
* Binary license management system

* add missing file

* add comment

* Address comments

* print missing licenses

* print druid module name

* Add missing licenses and update versions

* fix library versions and add missing ones. also fix pom.xml

* testing multi thread

* Parallel report generation

* fix build error

* install pyyaml and use old api

* install python3

* fix travis script

* python3.6

* pip

* setuptools

* python3-setuptools

* address comment

* error on not found reports or registered licenses

* removed licenses

* debug

* travis debug

* add missing licenses

* travis debug

* debug

* remove debug code

* test build script

* travis debug

* still debug

* add missing python lib

* debug

* debug

* fix travis

* fix travis

* debug travis

* flush print

* print something more to keep travis alive

* adjust print

* single threaded

* single threaded

* debug

* debug

* remove debug

* remove deprecated-2017Q4 from travis conf

* remove comments and duplicate sudo
  • Loading branch information
jihoonson committed Jul 8, 2019
1 parent f2e4e65 commit 12f12676e3fdb747807265ea909b94699f96bf02
Show file tree
Hide file tree
Showing 11 changed files with 5,457 additions and 1,264 deletions.
@@ -15,12 +15,8 @@

language: java

# On 12-12-2017, Travis updated their trusty image, which caused integration tests to fail.
# The group: config instructs Travis to use the previous trusty image.
# Please see https://github.com/apache/incubator-druid/pull/5155 for more information.
sudo: false
dist: trusty
group: deprecated-2017Q4
sudo: true
dist: xenial

jdk:
- openjdk8
@@ -50,6 +46,10 @@ matrix:
# packaging check
- env:
- NAME="packaging check"
before_install:
- sudo apt-get update && sudo apt-get install python3 python3-pip python3-setuptools -y
- pip3 install wheel # install wheel first explicitly
- pip3 install pyyaml
install: true
script: MAVEN_OPTS='-Xmx3000m' mvn -DskipTests -Dforbiddenapis.skip=true -Dcheckstyle.skip=true -Dpmd.skip=true -Dmaven.javadoc.skip=true -pl '!benchmarks' -B --fail-at-end clean install -Pdist -Pbundle-contrib-exts

@@ -117,8 +117,7 @@ matrix:
- free -m

# run integration tests
- sudo: required
services:
- services:
- docker
env:
- NAME="integration test part 1"
@@ -138,8 +137,7 @@ matrix:
done

# run integration tests
- sudo: required
services:
- services:
- docker
env:
- NAME="integration test part 2"

This file was deleted.

@@ -130,6 +130,37 @@
</arguments>
</configuration>
</execution>
<execution>
<id>generate-dependency-reports</id>
<phase>initialize</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<executable>${project.parent.basedir}/docs/_bin/generate-license-dependency-reports.py</executable>
<arguments>
<argument>${project.basedir}/../</argument>
<argument>${project.basedir}/target</argument>
<argument>--clean-maven-artifact-transfer</argument>
</arguments>
</configuration>
</execution>
<execution>
<id>verify-and-generate-license</id>
<phase>initialize</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<executable>${project.parent.basedir}/docs/_bin/generate-license.py</executable>
<arguments>
<argument>${project.basedir}/../licenses/APACHE2</argument>
<argument>${project.basedir}/../licenses.yaml</argument>
<argument>${project.basedir}/target/license-reports</argument>
<argument>${project.basedir}/../LICENSE.BINARY</argument>
</arguments>
</configuration>
</execution>
<execution>
<id>pull-deps</id>
<phase>package</phase>
@@ -16,62 +16,87 @@
# limitations under the License.

import os
import shutil
import subprocess
import sys
import argparse
import concurrent.futures
import time
import threading


existing_jar_dict_notice = {}
def generate_report(module_path, report_orig_path, report_out_path):
is_dir = os.path.isdir(module_path)
if not is_dir:
print("{} is not a directory".format(module_path))
return

def generate_reports(druid_path, tmp_path, exclude_ext):
license_main_path = tmp_path + "/license-reports"
license_ext_path = tmp_path + "/license-reports/ext"
os.mkdir(license_main_path)
os.mkdir(license_ext_path)
os.makedirs(report_out_path, exist_ok=True)

try:
print("Generating report for {}".format(module_path))
# This command prints lots of false errors. Here, we redirect stdout and stderr to avoid them.
command = "mvn -Ddependency.locations.enabled=false -Ddependency.details.enabled=false project-info-reports:dependencies"
subprocess.check_output(command, cwd=module_path, shell=True)
command = "cp -r {} {}".format(report_orig_path, report_out_path)
subprocess.check_output(command, cwd=module_path, shell=True)
print("Generated report for {} in {}".format(module_path, report_out_path))
except Exception as e:
print("Encountered error [{}] when generating report for {}".format(e, module_path))

print("********** Generating main LICENSE report.... **********")
os.chdir(druid_path)
command = "mvn -Pdist -Ddependency.locations.enabled=false project-info-reports:dependencies"
outstr = subprocess.check_output(command, shell=True).decode('UTF-8')
command = "cp -r distribution/target/site {}/site".format(license_main_path)
outstr = subprocess.check_output(command, shell=True).decode('UTF-8')

if exclude_ext:
sys.exit()
def generate_reports(druid_path, tmp_path, exclude_ext, num_threads):
tmp_path = os.path.abspath(tmp_path)
license_report_root = os.path.join(tmp_path, "license-reports")
license_core_path = os.path.join(license_report_root, "core")
license_ext_path = os.path.join(license_report_root, "ext")
shutil.rmtree(license_report_root, ignore_errors=True)
os.makedirs(license_core_path)
os.makedirs(license_ext_path)
druid_path = os.path.abspath(druid_path)

print("********** Generating extension LICENSE reports.... **********")
extension_dirs = os.listdir("extensions-core")
print("Found {}".format(extension_dirs))
for extension_dir in extension_dirs:
full_extension_dir = druid_path + "/extensions-core/" + extension_dir
if not os.path.isdir(full_extension_dir):
print("{} is not a directory".format(full_extension_dir))
continue
script_args = [(druid_path, os.path.join(druid_path, "distribution", "target", "site"), license_core_path)]

print("--- Generating report for {}... ---".format(extension_dir))
if not exclude_ext:
extensions_core_path = os.path.join(druid_path, "extensions-core")
extension_dirs = os.listdir(extensions_core_path)
print("Found {} extensions".format(len(extension_dirs)))
for extension_dir in extension_dirs:
print("extension dir: {}".format(extension_dir))
extension_path = os.path.join(extensions_core_path, extension_dir)
if not os.path.isdir(extension_path):
print("{} is not a directory".format(extension_path))
continue

extension_report_dir = "{}/{}".format(license_ext_path, extension_dir)
os.mkdir(extension_report_dir)
prev_work_dir = os.getcwd()
os.chdir(full_extension_dir)
extension_report_dir = "{}/{}".format(license_ext_path, extension_dir)
script_args.append((extension_path, os.path.join(extension_path, "target", "site"), extension_report_dir))

print("Generating dependency reports")

try:
command = "mvn -Ddependency.locations.enabled=false project-info-reports:dependencies"
outstr = subprocess.check_output(command, shell=True).decode('UTF-8')
command = "cp -r target/site {}/site".format(extension_report_dir)
outstr = subprocess.check_output(command, shell=True).decode('UTF-8')
except:
print("Encountered error when generating report for: " + extension_dir)
if num_threads > 1:
with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
for module_path, report_orig_path, report_out_path in script_args:
executor.submit(generate_report, module_path, report_orig_path, report_out_path)
else:
for module_path, report_orig_path, report_out_path in script_args:
generate_report(module_path, report_orig_path, report_out_path)

os.chdir(prev_work_dir)

if __name__ == "__main__":
try:
parser = argparse.ArgumentParser(description='Generating dependency reports.')
parser.add_argument('druid_path', metavar='<Druid source path>', type=str)
parser.add_argument('tmp_path', metavar='<Full tmp path>', type=str)
parser.add_argument('--exclude-extension', dest='exclude_ext', action='store_const', const=True, default=False, help="Exclude extension report")
parser.add_argument('--clean-maven-artifact-transfer', dest='clean_mvn_artifact_transfer', action='store_const', const=True, default=False, help="Clean maven-artifact-transfer before generating dependency reports")
parser.add_argument('--parallel', dest='num_threads', type=int, default=1, help='Number of threads for generating reports')
args = parser.parse_args()
generate_reports(args.druid_path, args.tmp_path, args.exclude_ext)

# The default maven-artifact-transfer in Travis is currently corrupted. Set the below argument properly to remove the corrupted one.
if args.clean_mvn_artifact_transfer:
command = "rm -rf ~/.m2/repository/org/apache/maven/shared/maven-artifact-transfer"
subprocess.check_call(command, shell=True)

generate_reports(args.druid_path, args.tmp_path, args.exclude_ext, args.num_threads)
except KeyboardInterrupt:
print('Interrupted, closing.')

0 comments on commit 12f1267

Please sign in to comment.