ICU-20066 add a copyright scan stage (unicode-org#26)

- name the travis build steps
- copy cpyskip.txt to /.cpyskip.txt
- add one named "lint" which runs cpyscan.pl
- remove network access from Cpy.pm (requires installation)

.cpyskip.txt

@@ -0,0 +1,103 @@
+#
+# Copyright (C) 2017 and later: Unicode, Inc. and others.
+# License & terms of use: http://www.unicode.org/copyright.html
+#
+# Copyright (c) 2005-2011 International Business Machines
+# Corporation and others. All Rights Reserved.
+## see NOTE below.
+## Also see: http://icu-project.org/copyright-scan.html
+#
+# git stuff
+# .git is ignored by code.
+#.git/*
+.gitignore
+.gitattributes
+.travis.yml
+.appveyor.yml
+# suffix matches - start with '*'.  They are turned into as RE, '.brk$'
+*.brk
+*.bz2
+*.classpath
+*.csproj
+*.cvsignore
+*.dat
+*.DS_Store
+*.doc
+*.gif
+*.gz
+*.ico
+*.icu
+*.intaglio
+*.jar
+*.jpg
+*.launch
+*.nrm
+*.odp
+*.otf
+*.pdf
+*.png
+*.ppt
+*.prefs
+*.project
+*.res
+*.rtf
+*.sln
+*.sxd
+*.sxg
+*.sxw
+*.tri2
+*.vcproj
+*.vcxproj
+*.vcxproj.filters
+*.zip
+
+# UnicodeData.txt does not have any header.
+*/UnicodeData.txt
+
+#
+# ICU4C
+#
+icu4c/source/aclocal.m4
+icu4c/source/config.guess
+icu4c/source/config.log
+icu4c/source/config.status
+icu4c/source/config.sub
+icu4c/source/install-sh
+icu4c/source/extra/uconv/samples/*
+icu4c/source/samples/layout/Sample.txt
+icu4c/source/samples/ucnv/data01.txt
+icu4c/source/samples/ufortune/resources/res-file-list.txt
+icu4c/source/test/testdata/ConverterSelectorTestUTF8.txt
+icu4c/source/test/testdata/encoded.utf16be
+icu4c/source/test/testdata/idna_conf.txt
+icu4c/source/test/testdata/ra.xlf
+icu4c/source/test/testdata/re_tests.txt
+icu4c/source/test/thaitest/space.txt
+icu4c/source/tools/tzcode/asctime.c
+icu4c/source/tools/tzcode/ialloc.c
+icu4c/source/tools/tzcode/localtime.c
+icu4c/source/tools/tzcode/private.h
+icu4c/source/tools/tzcode/scheck.c
+icu4c/source/tools/tzcode/tzfile.h
+icu4c/source/tools/tzcode/tzselect.ksh
+icu4c/source/tools/tzcode/zdump.c
+icu4c/source/tools/tzcode/zic.c
+#
+# ICU4J
+#
+icu4j/eclipse-build/pdebuild/allElements.xml
+icu4j/eclipse-build/pdebuild/customTargets.xml
+icu4j/main/tests/core/src/com/ibm/icu/dev/data/IDNATestInput.txt
+icu4j/main/tests/core/src/com/ibm/icu/dev/data/unicode/confusablesWholeScript.txt
+icu4j/main/tests/core/src/com/ibm/icu/dev/data/unicode/UnicodeData.txt
+icu4j/main/tests/core/src/com/ibm/icu/dev/test/duration/testdata/*
+icu4j/perf-tests/data/conversion/*
+#
+# tools
+#
+tools/trac/IcuCodeTools/*
+tools/unicodetools/*
+#
+# vendor
+#
+vendor/double-conversion/upstream/*

.travis.yml

@@ -1,6 +1,7 @@
 matrix:
   include:
-    - language: java
+    - name: "j"
+      language: java
       env:      BUILD=ICU4J
       before_script:
         - cd icu4j
@@ -10,12 +11,20 @@ matrix:
       after_failure:
           -  cat `find out/junit-results -name "*.txt" -exec grep -l FAILED {} \;`
 
-    - language: cpp
+    - name: "c: linux gcc"
+      language: cpp
       env:      BUILD=ICU4C_GCC
       compiler: gcc
       script:   cd icu4c/source && ./runConfigureICU Linux && make -j2 check
 
-    - language: cpp
+    - name: "c: linux clang"
+      language: cpp
+      env:      BUILD=ICU4C_CLANG
+      compiler: clang
+      script:   cd icu4c/source && ./runConfigureICU Linux && make -j2 check
+
+    - name: "c: osx clang"
+      language: cpp
       env:      BUILD=MACINTOSH
       os:       osx
       compiler: clang
@@ -24,7 +33,8 @@ matrix:
     # Clang Linux with address sanitizer.
     # Note - the 'sudo: true' option forces Travis to use a Virtual machine on GCE instead of
     #        a Container on EC2 or Packet. Asan builds of ICU fail otherwise.
-    - language: cpp
+    - name: "c: linux asan"
+      language: cpp
       env:      BUILD=ICU4C_CLANG_ASAN
       os:       linux
       dist:     trusty
@@ -43,7 +53,8 @@ matrix:
 
     # Clang Linux with thread sanitizer.
     #
-    - language: cpp
+    - name: "c: linux tsan"
+      language: cpp
       env:      BUILD=ICU4C_CLANG_THREAD_SAN
       os:       linux
       dist:     trusty
@@ -58,3 +69,8 @@ matrix:
                   - clang-5.0
       script:
           - cd icu4c/source && CPPFLAGS="-fsanitize=thread" LDFLAGS="-fsanitize=thread" ./runConfigureICU --enable-debug --disable-release Linux --disable-renaming && make -j2 check
+
+    # copyright scan / future linter
+    - name "lint"
+      script:
+          - perl tools/scripts/cpysearch/cpyscan.pl

tools/scripts/cpysearch/Cpy.pm

@@ -15,7 +15,6 @@ package Cpy;
 use strict;
 use warnings;
 use base 'Exporter';
-use LWP::Simple;
 
 our @EXPORT = qw(any glob_to_regex should_ignore);
 
@@ -81,19 +80,16 @@ sub glob_to_regex($) {
 }
 
 # Load cpyskip.txt contents.
-# Try local cpyskip.txt first - if not found, try online version
-our $cpyskip_file = "cpyskip.txt";
+# Try local .cpyskip.txt
+# no support for HTTP fetch.
+our $cpyskip_file = ".cpyskip.txt";
 our @cpyskip_lines;
 if (open(our $cpyskip_fh, "<", $cpyskip_file)) {
     @cpyskip_lines = <$cpyskip_fh>;
     close $cpyskip_fh;
-    print "Using local cpyskip.txt\n";
+    # print "Using local cpyskip.txt\n";
 } else {
-    our $cpyskip_url = "http://source.icu-project.org/cpyskip.txt";
-    our $cpyskip = get($cpyskip_url);
-    die "Can't get $cpyskip_url" if (! defined $cpyskip);
-    @cpyskip_lines = split(/\n/, $cpyskip);
-    print "Using " . $cpyskip_url . "\n";
+    die "Could not open $cpyskip_file";
 }
 our @ignore_globs = map  { chomp; glob_to_regex($_) }
                     grep { /^\s*[^#\s]+/ }

tools/scripts/cpysearch/cpyscan.pl

@@ -24,10 +24,16 @@
 
 use Cpy;
 my $icu_src = $ARGV[0] || ".";
+my $exitStatus = 0;
 my $icu_src_len = length($icu_src);
 die "Can't open ICU directory: $icu_src" unless -d $icu_src;
 find({
         wanted => sub {
+            # save a little bit of time.            
+            if ($_ eq './.git') {
+                $File::Find::prune = 1;
+                return;
+            }
             return unless -f;
             my $relpath = substr($_, $icu_src_len + 1);
             return if should_ignore($relpath);
@@ -36,8 +42,15 @@
             my $result = any { $_ =~ /(Copyright|©).*Unicode/i } <F>;
 
             close F;
-
-            print "$relpath\n" unless $result;
+            if (not $result) {
+                print "$relpath\n";
+                $exitStatus = 1;
+            }
         },
         no_chdir => 1,
     }, $icu_src);
+
+if ($exitStatus) {
+    die "Above files did not contain the correct copyright notice.";
+}
+exit $exitStatus;