Add tests for system package licenses #478

Validate that things run OK and license are detected with a minimal apine and debian docker image. Also adapt existing tests to new results. Bump SCTK to latest version. Reference: #478 Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>
nexB · Aug 1, 2022 · 58fa264 · 58fa264
1 parent 97bb941
commit 58fa264
Show file tree

Hide file tree

Showing 9 changed files with 3,746 additions and 50 deletions.
diff --git a/scanpipe/tests/data/alpine_3_15_4_scan_codebase.json b/scanpipe/tests/data/alpine_3_15_4_scan_codebase.json
@@ -117,7 +117,7 @@
       "code_view_url": "",
       "vcs_url": "git+http://git.alpinelinux.org/aports/commit/?id=dfa1379357a321e638feef1cd8d55ab03d020f45",
       "copyright": "",
-      "license_expression": "",
+      "license_expression": "gpl-2.0",
       "declared_license": "GPL-2.0-only",
       "notice_text": "",
       "manifest_path": "",
@@ -186,7 +186,7 @@
       "code_view_url": "",
       "vcs_url": "git+http://git.alpinelinux.org/aports/commit/?id=aab68f8c9ab434a46710de8e12fb3206e2930a59",
       "copyright": "",
-      "license_expression": "",
+      "license_expression": "mit",
       "declared_license": "MIT",
       "notice_text": "",
       "manifest_path": "",
@@ -375,7 +375,7 @@
       "code_view_url": "",
       "vcs_url": "git+http://git.alpinelinux.org/aports/commit/?id=1ac3c1bb29eeff083c621cf6b27ad12ab93cb73a",
       "copyright": "",
-      "license_expression": "",
+      "license_expression": "gpl-2.0",
       "declared_license": "GPL-2.0-only",
       "notice_text": "",
       "manifest_path": "",
@@ -409,7 +409,7 @@
       "code_view_url": "",
       "vcs_url": "git+http://git.alpinelinux.org/aports/commit/?id=2745de7e1b09e663b477a8141b84f7d81a049963",
       "copyright": "",
-      "license_expression": "",
+      "license_expression": "gpl-2.0",
       "declared_license": "GPL-2.0-only",
       "notice_text": "",
       "manifest_path": "",
@@ -454,7 +454,7 @@
       "code_view_url": "",
       "vcs_url": "git+http://git.alpinelinux.org/aports/commit/?id=709b70bcb72738cfedc510bba08141b012038167",
       "copyright": "",
-      "license_expression": "",
+      "license_expression": "mpl-2.0 AND mit",
       "declared_license": "MPL-2.0 AND MIT",
       "notice_text": "",
       "manifest_path": "",
@@ -499,7 +499,7 @@
       "code_view_url": "",
       "vcs_url": "git+http://git.alpinelinux.org/aports/commit/?id=455e966899a9358fc94f5bce633afe8a1942095c",
       "copyright": "",
-      "license_expression": "",
+      "license_expression": "openssl-ssleay",
       "declared_license": "OpenSSL",
       "notice_text": "",
       "manifest_path": "",
@@ -568,7 +568,7 @@
       "code_view_url": "",
       "vcs_url": "git+http://git.alpinelinux.org/aports/commit/?id=60424133be2e79bbfeff3d58147a22886f817ce2",
       "copyright": "",
-      "license_expression": "",
+      "license_expression": "bsd-simplified AND bsd-new",
       "declared_license": "BSD-2-Clause AND BSD-3-Clause",
       "notice_text": "",
       "manifest_path": "",
@@ -602,7 +602,7 @@
       "code_view_url": "",
       "vcs_url": "git+http://git.alpinelinux.org/aports/commit/?id=91c7a9f3aa296b6d462c5634e7658ebdbff65bb9",
       "copyright": "",
-      "license_expression": "",
+      "license_expression": "isc AND (bsd-new OR mit)",
       "declared_license": "ISC AND (BSD-3-Clause OR MIT)",
       "notice_text": "",
       "manifest_path": "",
@@ -647,7 +647,7 @@
       "code_view_url": "",
       "vcs_url": "git+http://git.alpinelinux.org/aports/commit/?id=455e966899a9358fc94f5bce633afe8a1942095c",
       "copyright": "",
-      "license_expression": "",
+      "license_expression": "openssl-ssleay",
       "declared_license": "OpenSSL",
       "notice_text": "",
       "manifest_path": "",
@@ -692,7 +692,7 @@
       "code_view_url": "",
       "vcs_url": "git+http://git.alpinelinux.org/aports/commit/?id=bf5bbfdbf780092f387b7abe401fbfceda90c84d",
       "copyright": "",
-      "license_expression": "",
+      "license_expression": "mit",
       "declared_license": "MIT",
       "notice_text": "",
       "manifest_path": "",
@@ -737,7 +737,7 @@
       "code_view_url": "",
       "vcs_url": "git+http://git.alpinelinux.org/aports/commit/?id=bf5bbfdbf780092f387b7abe401fbfceda90c84d",
       "copyright": "",
-      "license_expression": "",
+      "license_expression": "mit AND bsd-new AND gpl-2.0-plus",
       "declared_license": "MIT BSD GPL2+",
       "notice_text": "",
       "manifest_path": "",
@@ -771,7 +771,7 @@
       "code_view_url": "",
       "vcs_url": "git+http://git.alpinelinux.org/aports/commit/?id=86b3d4fbb0a760febf3476f9a58abf8d0f728d5c",
       "copyright": "",
-      "license_expression": "",
+      "license_expression": "gpl-2.0",
       "declared_license": "GPL-2.0-only",
       "notice_text": "",
       "manifest_path": "",
@@ -805,7 +805,7 @@
       "code_view_url": "",
       "vcs_url": "git+http://git.alpinelinux.org/aports/commit/?id=2745de7e1b09e663b477a8141b84f7d81a049963",
       "copyright": "",
-      "license_expression": "",
+      "license_expression": "gpl-2.0",
       "declared_license": "GPL-2.0-only",
       "notice_text": "",
       "manifest_path": "",
@@ -839,7 +839,7 @@
       "code_view_url": "",
       "vcs_url": "git+http://git.alpinelinux.org/aports/commit/?id=74148808679f47ad96dc99e83ef73acfdeec1642",
       "copyright": "",
-      "license_expression": "",
+      "license_expression": "zlib",
       "declared_license": "Zlib",
       "notice_text": "",
       "manifest_path": "",
@@ -1643,7 +1643,7 @@
       "for_packages": [
         "pkg:alpine/alpine-baselayout@3.2.0-r18?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758"
       ],
-      "path": "alpine_3_15_4.tar.gz-extract/40e48c8ef2450e6a9e8d50b846a58ede43f1b01dd351d2bdd7dca14c5c033f20/etc/profile.d/color_prompt.sh.disabled",
+      "path": "alpine_3_15_4.tar.gz-extract/40e48c8ef2450e6a9e8d50b846a58ede43f1b01dd351d2bdd7dca14c5c033f20/etc/profile.d/README",
       "sha1": "",
       "md5": "",
       "extra_data": {},
@@ -1657,8 +1657,8 @@
       "status": "system-package",
       "tag": "img-06c7c4-layer-01-40e48c",
       "type": "file",
-      "name": "color_prompt.sh",
-      "extension": ".disabled",
+      "name": "README",
+      "extension": "",
       "programming_language": "",
       "mime_type": "inode/x-empty",
       "is_binary": false,
@@ -1671,7 +1671,7 @@
       "for_packages": [
         "pkg:alpine/alpine-baselayout@3.2.0-r18?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758"
       ],
-      "path": "alpine_3_15_4.tar.gz-extract/40e48c8ef2450e6a9e8d50b846a58ede43f1b01dd351d2bdd7dca14c5c033f20/etc/profile.d/locale.sh",
+      "path": "alpine_3_15_4.tar.gz-extract/40e48c8ef2450e6a9e8d50b846a58ede43f1b01dd351d2bdd7dca14c5c033f20/etc/profile.d/color_prompt.sh.disabled",
       "sha1": "",
       "md5": "",
       "extra_data": {},
@@ -1685,9 +1685,9 @@
       "status": "system-package",
       "tag": "img-06c7c4-layer-01-40e48c",
       "type": "file",
-      "name": "locale",
-      "extension": ".sh",
-      "programming_language": "Bash",
+      "name": "color_prompt.sh",
+      "extension": ".disabled",
+      "programming_language": "",
       "mime_type": "inode/x-empty",
       "is_binary": false,
       "is_text": true,
@@ -1699,7 +1699,7 @@
       "for_packages": [
         "pkg:alpine/alpine-baselayout@3.2.0-r18?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758"
       ],
-      "path": "alpine_3_15_4.tar.gz-extract/40e48c8ef2450e6a9e8d50b846a58ede43f1b01dd351d2bdd7dca14c5c033f20/etc/profile.d/README",
+      "path": "alpine_3_15_4.tar.gz-extract/40e48c8ef2450e6a9e8d50b846a58ede43f1b01dd351d2bdd7dca14c5c033f20/etc/profile.d/locale.sh",
       "sha1": "",
       "md5": "",
       "extra_data": {},
@@ -1713,9 +1713,9 @@
       "status": "system-package",
       "tag": "img-06c7c4-layer-01-40e48c",
       "type": "file",
-      "name": "README",
-      "extension": "",
-      "programming_language": "",
+      "name": "locale",
+      "extension": ".sh",
+      "programming_language": "Bash",
       "mime_type": "inode/x-empty",
       "is_binary": false,
       "is_text": true,
@@ -1865,9 +1865,9 @@
     },
     {
       "for_packages": [
-        "pkg:alpine/libcrypto1.1@1.1.1n-r0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758"
+        "pkg:alpine/ca-certificates-bundle@20211220-r0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758"
       ],
-      "path": "alpine_3_15_4.tar.gz-extract/40e48c8ef2450e6a9e8d50b846a58ede43f1b01dd351d2bdd7dca14c5c033f20/etc/ssl1.1/ct_log_list.cnf.dist",
+      "path": "alpine_3_15_4.tar.gz-extract/40e48c8ef2450e6a9e8d50b846a58ede43f1b01dd351d2bdd7dca14c5c033f20/etc/ssl/certs/ca-certificates.crt",
       "sha1": "",
       "md5": "",
       "extra_data": {},
@@ -1881,8 +1881,8 @@
       "status": "system-package",
       "tag": "img-06c7c4-layer-01-40e48c",
       "type": "file",
-      "name": "ct_log_list.cnf",
-      "extension": ".dist",
+      "name": "ca-certificates",
+      "extension": ".crt",
       "programming_language": "",
       "mime_type": "inode/x-empty",
       "is_binary": false,
@@ -1895,7 +1895,7 @@
       "for_packages": [
         "pkg:alpine/libcrypto1.1@1.1.1n-r0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758"
       ],
-      "path": "alpine_3_15_4.tar.gz-extract/40e48c8ef2450e6a9e8d50b846a58ede43f1b01dd351d2bdd7dca14c5c033f20/etc/ssl1.1/openssl.cnf",
+      "path": "alpine_3_15_4.tar.gz-extract/40e48c8ef2450e6a9e8d50b846a58ede43f1b01dd351d2bdd7dca14c5c033f20/etc/ssl1.1/ct_log_list.cnf.dist",
       "sha1": "",
       "md5": "",
       "extra_data": {},
@@ -1909,8 +1909,8 @@
       "status": "system-package",
       "tag": "img-06c7c4-layer-01-40e48c",
       "type": "file",
-      "name": "openssl",
-      "extension": ".cnf",
+      "name": "ct_log_list.cnf",
+      "extension": ".dist",
       "programming_language": "",
       "mime_type": "inode/x-empty",
       "is_binary": false,
@@ -1923,7 +1923,7 @@
       "for_packages": [
         "pkg:alpine/libcrypto1.1@1.1.1n-r0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758"
       ],
-      "path": "alpine_3_15_4.tar.gz-extract/40e48c8ef2450e6a9e8d50b846a58ede43f1b01dd351d2bdd7dca14c5c033f20/etc/ssl1.1/openssl.cnf.dist",
+      "path": "alpine_3_15_4.tar.gz-extract/40e48c8ef2450e6a9e8d50b846a58ede43f1b01dd351d2bdd7dca14c5c033f20/etc/ssl1.1/openssl.cnf",
       "sha1": "",
       "md5": "",
       "extra_data": {},
@@ -1937,8 +1937,8 @@
       "status": "system-package",
       "tag": "img-06c7c4-layer-01-40e48c",
       "type": "file",
-      "name": "openssl.cnf",
-      "extension": ".dist",
+      "name": "openssl",
+      "extension": ".cnf",
       "programming_language": "",
       "mime_type": "inode/x-empty",
       "is_binary": false,
@@ -1949,9 +1949,9 @@
     },
     {
       "for_packages": [
-        "pkg:alpine/ca-certificates-bundle@20211220-r0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758"
+        "pkg:alpine/libcrypto1.1@1.1.1n-r0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758"
       ],
-      "path": "alpine_3_15_4.tar.gz-extract/40e48c8ef2450e6a9e8d50b846a58ede43f1b01dd351d2bdd7dca14c5c033f20/etc/ssl/certs/ca-certificates.crt",
+      "path": "alpine_3_15_4.tar.gz-extract/40e48c8ef2450e6a9e8d50b846a58ede43f1b01dd351d2bdd7dca14c5c033f20/etc/ssl1.1/openssl.cnf.dist",
       "sha1": "",
       "md5": "",
       "extra_data": {},
@@ -1965,8 +1965,8 @@
       "status": "system-package",
       "tag": "img-06c7c4-layer-01-40e48c",
       "type": "file",
-      "name": "ca-certificates",
-      "extension": ".crt",
+      "name": "openssl.cnf",
+      "extension": ".dist",
       "programming_language": "",
       "mime_type": "inode/x-empty",
       "is_binary": false,

diff --git a/scanpipe/tests/data/basic-rootfs_root_filesystems.json b/scanpipe/tests/data/basic-rootfs_root_filesystems.json
@@ -98,8 +98,8 @@
       "bug_tracking_url": "",
       "code_view_url": "",
       "vcs_url": "",
-      "copyright": "",
-      "license_expression": "",
+      "copyright": "Copyright (c) 1998-2016 Free Software Foundation, Inc.\nCopyright (c) 2001 by Pradeep Padala\nCopyright (c) 1994 X Consortium\nCopyright (c) 1980, 1991, 1992, 1993 The Regents of the University of California\nCopyright 1996-2007 by Thomas E. Dickey",
+      "license_expression": "x11-fsf AND x11-xconsortium AND bsd-new AND x11-fsf",
       "declared_license": "",
       "notice_text": "",
       "manifest_path": "",
@@ -170,9 +170,9 @@
       "bug_tracking_url": "",
       "code_view_url": "",
       "vcs_url": "",
-      "copyright": "",
-      "license_expression": "",
-      "declared_license": "",
+      "copyright": "Copyright 2013 Jiri Pirko <jiri@resnulli.us>\nCopyright 2014 Andrew Ayer <agwa@andrewayer.name>",
+      "license_expression": "(lgpl-2.1-plus AND lgpl-2.1-plus AND lgpl-2.1) AND (lgpl-2.1-plus AND lgpl-2.1-plus AND lgpl-2.1)",
+      "declared_license": "['LGPL-2.1+', 'LGPL-2.1+', 'LGPL-2.1+']",
       "notice_text": "",
       "manifest_path": "",
       "contains_source_code": null,

diff --git a/scanpipe/tests/data/debian_scan_codebase.json b/scanpipe/tests/data/debian_scan_codebase.json
@@ -116,8 +116,8 @@
       "bug_tracking_url": "",
       "code_view_url": "",
       "vcs_url": "",
-      "copyright": "",
-      "license_expression": "",
+      "copyright": "Copyright (c) 1998-2016 Free Software Foundation, Inc.\nCopyright (c) 2001 by Pradeep Padala\nCopyright (c) 1994 X Consortium\nCopyright (c) 1980, 1991, 1992, 1993 The Regents of the University of California\nCopyright 1996-2007 by Thomas E. Dickey",
+      "license_expression": "x11-fsf AND x11-xconsortium AND bsd-new AND x11-fsf",
       "declared_license": "",
       "notice_text": "",
       "manifest_path": "",
@@ -188,9 +188,9 @@
       "bug_tracking_url": "",
       "code_view_url": "",
       "vcs_url": "",
-      "copyright": "",
-      "license_expression": "",
-      "declared_license": "",
+      "copyright": "Copyright 2013 Jiri Pirko <jiri@resnulli.us>\nCopyright 2014 Andrew Ayer <agwa@andrewayer.name>",
+      "license_expression": "(lgpl-2.1-plus AND lgpl-2.1-plus AND lgpl-2.1) AND (lgpl-2.1-plus AND lgpl-2.1-plus AND lgpl-2.1)",
+      "declared_license": "['LGPL-2.1+', 'LGPL-2.1+', 'LGPL-2.1+']",
       "notice_text": "",
       "manifest_path": "",
       "contains_source_code": null,

diff --git a/scanpipe/tests/data/docker-mini-with-license-alpine.tar.xz b/scanpipe/tests/data/docker-mini-with-license-alpine.tar.xz