From 6fcad1165e3e8a041191c3b516275f3d70bc9289 Mon Sep 17 00:00:00 2001 From: ma-ilsi Date: Mon, 25 Sep 2023 11:42:47 -0400 Subject: [PATCH] Fix copyright linter Linter now has a complete list of files that were once missing. More files can easily be added with a regex pattern. -scan flag allows finding new files that have not been registered. --- codebuild/bin/copyright_mistake_scanner.sh | 224 +++++++++++++++++++-- 1 file changed, 212 insertions(+), 12 deletions(-) diff --git a/codebuild/bin/copyright_mistake_scanner.sh b/codebuild/bin/copyright_mistake_scanner.sh index 662d7a0236f..e94bd9f4198 100755 --- a/codebuild/bin/copyright_mistake_scanner.sh +++ b/codebuild/bin/copyright_mistake_scanner.sh @@ -13,30 +13,230 @@ # permissions and limitations under the License. # -set -e -S2N_FILES=$(find "$PWD" -type f -name "s2n_*.[ch]") -S2N_FILES+=" " -S2N_FILES+=$(find "$PWD"/codebuild/ -type f -name "*.sh") -S2N_FILES+=" " -S2N_FILES+=$(find "$PWD"/tests/ -type f -name "*.sh") +# NOTE: If you use the pipe character (|) anywhere in this string, you must escape it (\|). +# NOTE: If you use the double quote character (") anywhere in this string, you must escape it (\"), +# NOTE: End every pattern with the $ anchor, to be explicit. This is needed for the scan feature to correctly identify the files you already registered. +# You can safely comment in this string by using # and ending it with a newline character. -FAILED=0 +S2N_REGISTERED_PATTERNS=" + +#the license +.*s2n-tls/NOTICE$ + +#all the headers in /api +.*s2n-tls/api/.*\.h + +#all files in bin +.*s2n-tls/bin/[^/]*$ + +#all cpp files that begin with s2n_ +.*s2n-tls/.*/s2n_[^/]*\.cc$ + +#all c files that begin with s2n_ +.*s2n-tls/.*/s2n_[^/]*\.[ch]$ + +#all c files that begin with S2N_ +.*s2n-tls/.*/S2N_[^/]*\.[ch]$ + +#codebuild/bin shell scripts +.*s2n-tls/codebuild/bin/[^/]*.\.sh$ + +#that one C file in codebuild/bin +.*s2n-tls/codebuild/bin/s2n_dynamic_load_test\.c$ + +#codebuild/spec yamls +.*s2n-tls/codebuild/spec/[^/]*\.yml$ + +#all files in crypto +.*s2n-tls/crypto/[^/]*$ + +#that one docker yaml +.*s2n-tls/docker-images/docker-compose\.yml$ + +#saw files +.*s2n-tls/tests/saw/.*\.saw$ + +#all files in tests/benchmark/utils +.*s2n-tls/tests/benchmark/utils/.*$ + +#cry files in tests/saw/HMAC/spec +.*s2n-tls/tests/saw/HMAC/spec/[^/]*\.cry + +#tests/cbmc/stubs (unregistered the README) +.*s2n-tls/tests/cbmc/stubs/[^/]*$ + +#tests/sidetrail/working/stubs +.*s2n-tls/tests/cbmc/stubs/[^/]*$ + +#all rust files in bindings/rust +.*s2n-tls/bindings/rust/.*[^/]*\.rs$ + +#all shell scripts in bindings/rust +.*s2n-tls/bindings/rust/.*[^/]*\.sh$ + +#cbmc_proof headers +.*s2n-tls/tests/cbmc/include/cbmc_proof/.*$ + +#python files in test/cbmc/proofs +.*/s2n-tls/tests/cbmc/proofs/.*\.py$ + +#shell scripts in tests/sidetrail/working/s2n- +.*s2n-tls/tests/sidetrail/working/s2n-.*/.*[^/]\.sh$ + + +#makefiles +.*s2n-tls/Makefile$ +.*s2n-tls/stuffer/Makefile$ +.*s2n-tls/bin/Makefile$ +.*s2n-tls/tests/unit/Makefile$ +.*s2n-tls/tests/integrationv2/Makefile$ +.*s2n-tls/tests/saw/Makefile$ +.*s2n-tls/tests/benchmark/Makefile$ +.*s2n-tls/tests/cbmc/proofs/s2n_stuffer_erase_and_read_bytes/Makefile$ +.*s2n-tls/tests/cbmc/proofs/s2n_stuffer_dhparams_from_pem/Makefile$ +.*s2n-tls/tests/cbmc/proofs/s2n_stuffer_growable_alloc/Makefile$ +.*s2n-tls/tests/cbmc/proofs/Makefile.cbmc_batch$ +.*s2n-tls/tests/cbmc/proofs/s2n_stuffer_certificate_from_pem/Makefile$ +.*s2n-tls/tests/cbmc/proofs/s2n_stuffer_private_key_from_pem/Makefile$ +.*s2n-tls/tests/cbmc/proofs/s2n_stuffer_alloc_ro_from_string/Makefile$ +.*s2n-tls/tests/cbmc/proofs/s2n_stuffer_alloc/Makefile$ +.*s2n-tls/tests/cbmc/proofs/s2n_stuffer_read/Makefile$ +.*s2n-tls/tests/Makefile$ +.*s2n-tls/tests/ctverif/Makefile$ +.*s2n-tls/tests/viz/Makefile$ +.*s2n-tls/tests/testlib/Makefile$ +.*s2n-tls/tests/fuzz/Makefile$ +.*s2n-tls/tests/fuzz/LD_PRELOAD/Makefile$ +.*s2n-tls/tests/LD_PRELOAD/Makefile$ +.*s2n-tls/utils/Makefile$ +.*s2n-tls/tls/Makefile$ +.*s2n-tls/tls/extensions/Makefile$ +.*s2n-tls/lib/Makefile$ +.*s2n-tls/error/Makefile$ +.*s2n-tls/coverage/Makefile$ +.*s2n-tls/pq-crypto/Makefile$ +.*s2n-tls/pq-crypto/kyber_r3/Makefile$ +.*s2n-tls/pq-crypto/kyber_r3/Makefile$ + + +#files that can't be grouped well in regex: +.*s2n-tls/tests/cbmc/sources/cbmc_utils\.c$ +.*s2n-tls/tests/cbmc/sources/make_common_datastructures\.c$ +.*s2n-tls/tests/sidetrail/working/s2n-cbc/cbc\.c$ +.*s2n-tls/tests/sidetrail/count_success\.pl$ +.*s2n-tls/tests/ctverif/count_success\.pl$ +.*s2n-tls/tests/pems/sni/generate\.sh$ +.*s2n-tls/tests/pems/gen_self_signed_cert\.sh$ +.*s2n-tls/tests/pems/crl/test_s2nc\.sh$ +.*s2n-tls/tests/pems/crl/generate\.sh$ +.*s2n-tls/tests/pems/ocsp/generate\.sh$ +.*s2n-tls/tests/fuzz/runFuzzTest\.sh$ +.*s2n-tls/tests/fuzz/calcTotalCov\.sh$ +.*s2n-tls/tests/fuzz/LD_PRELOAD/global_overrides\.c$ +.*s2n-tls/tests/LD_PRELOAD/allocator_overrides\.c$ +.*s2n-tls/tests/saw/spec/handshake/rfc_handshake_tls13.cry$ +.*s2n-tls/tests/saw/spec/DRBG/AES.cry$ +.*tests/saw/spec/DRBG/DRBG.cry$ +.*s2n-tls/tests/saw/spec/extras/HMAC/LICENSE$ +.*s2n-tls/\.github/s2n_doxygen\.sh$ +.*s2n-tls/\.github/workflows/proof_ci\.yaml$ +.*s2n-tls/\.github/install_osx_dependencies\.sh$ +.*s2n-tls/\.github/s2n_bsd\.sh$ +.*s2n-tls/\.github/s2n_osx\.sh$ +.*s2n-tls/\.github/gha_monitor/gha_monitor/__main__\.py$ +.*s2n-tls/\.github/gha_monitor/gha_monitor/sns\.py$ +.*s2n-tls/bindings/rust/s2n-tls-sys/templates/features\.template$ +.*s2n-tls/.git/hooks/pre-rebase\.sample$ +.*s2n-tls/pq-crypto/s2n_pq_asm\.mk$ +.*s2n-tls/pq-crypto/kyber_r3/KeccakP-brg_endian_avx2\.h$ +" +#END OF REGISTERED PATTERNS + + +# NOTE: If you use the pipe character (|) anywhere in this string, you must escape it (\|). +# NOTE: If you use the double quote character (") anywhere in this string, you must escape it (\"), +# NOTE: End every spattern with the $ anchor, to be explicit. This is needed for the scan feature to correctly identify the files you already registered. +# You can safely comment in this string by using # and ending it with a newline character. + +S2N_UNREGISTERED_PATTERNS=" + +.*s2n-tls/tests/cbmc/stubs/README.md$ + +" +# END OF UNREGISTERED PATTERNS + +# Strip comments from pattern strings +S2N_REGISTERED_PATTERNS=`printf "%s" "$S2N_REGISTERED_PATTERNS" | sed "s|^#.*$||g"` +S2N_UNREGISTERED_PATTERNS=`printf "%s" "$S2N_UNREGISTERED_PATTERNS" | sed "s|^#.*$||g"` + + + +FAIL_COUNT=0 +SUCCESS_COUNT=0 + +# If the -scan flag is provided, we search all files not registered/unregistered and report on any new copyright headers that should be registered/unregistered. + +if [ "$1" == "-scan" ]; +then + ALL_FILES=`find "$PWD" -type f` + + for pattern in $S2N_REGISTERED_PATTERNS; do + ALL_FILES=`printf "%s" "$ALL_FILES" | sed "s|$pattern||g"` + done + + for pattern in $S2N_UNREGISTERED_PATTERNS; do + ALL_FILES=`printf "%s" "$ALL_FILES" | sed "s|$pattern||g"` + done + + for file in $ALL_FILES; do + # The word "Copyright" should appear at least once in the first 3 lines of every file + if head -3 "$file" | grep -q "Copyright"; + then + (( ++SUCCESS_COUNT )) + file=`printf "%s" "$file" | grep -o "s2n-tls/.*"` + printf "\nNew copyright header found:\n%s\n" "$file" + fi + done + + printf "\n%d new copyright headers found during scan.\n" "$SUCCESS_COUNT" + exit 0 +fi + + + +for pattern in $S2N_REGISTERED_PATTERNS; do + FOUND_FILES=`find "$PWD" -type f -regex "$pattern"` + S2N_FILES=`printf "%s\n%s" "$S2N_FILES" "$FOUND_FILES"` +done + +S2N_FILES=`printf "%s" "$S2N_FILES" | sort -u` + +for pattern in $S2N_UNREGISTERED_PATTERNS; do + S2N_FILES=`printf "%s" "$S2N_FILES" | sed "s|$pattern||g"` +done for file in $S2N_FILES; do # The word "Copyright" should appear at least once in the first 3 lines of every file - COUNT=`head -3 $file | grep "Copyright" | wc -l`; - if [ "$COUNT" == "0" ]; + if head -3 "$file" | grep -q "Copyright"; then - FAILED=1; - echo "Copyright Check Failed: $file"; + (( ++SUCCESS_COUNT )) + else + (( ++FAIL_COUNT )) + file=`printf "%s" "$file" | grep -o "s2n-tls/.*"` + printf "\n%s\n%s\n" "Copyright Check Failed:" "$file" fi done -if [ $FAILED == 1 ]; +TOTAL_COUNT=`expr $FAIL_COUNT + $SUCCESS_COUNT` + +printf "\n%d/%d Files Passing\n" "$SUCCESS_COUNT" "$TOTAL_COUNT" + +if [ $FAIL_COUNT -gt 0 ]; then printf "\\033[31;1mFAILED Copyright Check\\033[0m\\n" exit -1 else printf "\\033[32;1mPASSED Copyright Check\\033[0m\\n" + exit 0 fi