Spdx license expressions (#859)

* Add parser combinators to parse SPDX license expressions

The most important parser is `compoundExpression` it parses any SPDX
license expression into a list of SPDX simple-expressions.
The rationale is Nixpkgs' license metadata isn't capable of
distinguishing between the AND and OR relationships.
License exceptions aren't currently taken into account.

* Add tests for the SPDX parser combinators

I simply added a file with expressions that are expected to fail to
parse or parse successfully.

* Add the SPDX license list

The SPDX license list as attrsets following the nixpkgs lib.licenses
convention.

This uses fetchurl which is not ideal because it's not available during
pure evaluation.

It does not yet include the SPDX exceptions list.

* Refactor cabal-licenses.nix to use spdx/licenses.nix

The handling of the generic licenses is undecided still. Some have been
removed because they have better official identifiers.

* Refactor license mapping in builders

The common code in the comp- and setup-builders has been extracted and
refactored to use the SPDX expression parser.

* Use spdx-license-list-data from nixpkgs

This conveniently solves the impurity problem with using fetchurl : )
I'm not sure threading `pkgs` through everything to get access to the
spdx license list package is the right way to go about this.

* hscolour to "LGPL-2.1-only" and remove "LGPL"

* Use evalPackages for spdx and move shim to overlay

* Better fix for LGPL packages.

Co-authored-by: Hamish Mackenzie <Hamish.Mackenzie@iohk.io>

builder/comp-builder.nix

@@ -1,4 +1,4 @@
-{ stdenv, buildPackages, ghc, lib, gobject-introspection ? null, haskellLib, makeConfigFiles, haddockBuilder, ghcForComponent, hsPkgs, compiler, runCommand, libffi, gmp, zlib, ncurses, numactl, nodejs }:
+{ pkgs, stdenv, buildPackages, ghc, lib, gobject-introspection ? null, haskellLib, makeConfigFiles, haddockBuilder, ghcForComponent, hsPkgs, compiler, runCommand, libffi, gmp, zlib, ncurses, numactl, nodejs }:
 lib.makeOverridable (
 let self =
 { componentId
@@ -295,11 +295,7 @@ let
     meta = {
       homepage = package.homepage or "";
       description = package.synopsis or "";
-      license =
-        let
-          license-map = import ../lib/cabal-licenses.nix lib;
-        in license-map.${package.license} or
-          (builtins.trace "WARNING: license \"${package.license}\" not found" license-map.LicenseRef-OtherLicense);
+      license = haskellLib.cabalToNixpkgsLicense package.license;
       platforms = if platforms == null then stdenv.lib.platforms.all else platforms;
     };
 

builder/setup-builder.nix

@@ -1,4 +1,4 @@
-{ stdenv, lib, buildPackages, haskellLib, ghc, nonReinstallablePkgs, hsPkgs, makeSetupConfigFiles, pkgconfig }:
+{ pkgs, stdenv, lib, buildPackages, haskellLib, ghc, nonReinstallablePkgs, hsPkgs, makeSetupConfigFiles, pkgconfig }:
 
 { component, package, name, src, flags ? {}, revision ? null, patches ? [], defaultSetupSrc
 , preUnpack ? component.preUnpack, postUnpack ? component.postUnpack
@@ -55,11 +55,7 @@ let
       meta = {
         homepage = package.homepage or "";
         description = package.synopsis or "";
-        license =
-          let
-            license-map = import ../lib/cabal-licenses.nix lib;
-          in license-map.${package.license} or
-            (builtins.trace "WARNING: license \"${package.license}\" not found" license-map.LicenseRef-OtherLicense);
+        license = haskellLib.cabalToNixpkgsLicense package.license;
         platforms = if component.platforms == null then stdenv.lib.platforms.all else component.platforms;
       };
 

lib/cabal-licenses.nix

@@ -1,24 +1,20 @@
-lib: with lib.licenses;
-{ BSD-3-Clause = bsd3;
-  BSD-2-Clause = bsd2;
-  MIT = mit;
-  "MPL-2.0" = mpl20;
-  ISC = isc;
-  "LGPL-2.1-only"     = lgpl21;
-  "LGPL-3.0-only"     = lgpl3;
-  "GPL-2.0-only"      = gpl2;
-  "GPL-2.0-or-later"  = gpl2Plus;
-  "GPL-3.0-only"      = gpl3;
-  "AGPL-3.0-only"     = agpl3;
-  "AGPL-3.0-or-later" = agpl3Plus;
-  "Apache-2.0"        = asl20;
-  "GPL-2.0-or-later AND BSD-3-Clause" = [gpl2Plus bsd3];
-  "NCSA" = ncsa;
+pkgs:
+let licenses = import spdx/licenses.nix pkgs;
+in licenses // {
   # Generic
-  LicenseRef-Apache = "Apache";
-  LicenseRef-GPL = "GPL";
-  LicenseRef-LGPL = "LGPL";
-  LicenseRef-PublicDomain = publicDomain;
-  LicenseRef-OtherLicense = null;
+  LicenseRef-PublicDomain = {
+      spdxId = "LicenseRef-PublicDomain";
+      shortName = "Public Domain";
+      fullName = "This work is dedicated to the Public Domain";
+      url = "https://wikipedia.org/wiki/Public_domain";
+      free = true;
+    };
+  LicenseRef-OtherLicense = {
+      spdxId = "LicenseRef-OtherLicense";
+      shortName = "Other License";
+      fullName = "Unidentified Other License";
+      url = "https://spdx.github.io/spdx-spec/appendix-IV-SPDX-license-expressions/";
+      free = false;
+    };
   NONE = null;
 }

lib/default.nix

@@ -274,6 +274,9 @@ in {
     inherit pkgs;
   }) parseIndexState parseBlock;
 
+
+  cabalToNixpkgsLicense = import ./spdx/cabal.nix pkgs;
+
   # This function is like
   #   `src + (if subDir == "" then "" else "/" + subDir)`
   # however when `includeSiblings` is set it maintains

lib/spdx/cabal.nix

@@ -0,0 +1,19 @@
+let
+  spdx = import ./parser.nix;
+in pkgs:
+with builtins;
+let
+  # For better performance these are not in the
+  # let block below (probably helps by increasing
+  # the sharing)
+  license-map = import ../cabal-licenses.nix pkgs;
+  otherLicenseWarning = lic:
+    trace "WARNING: license \"${lic}\" not found"
+          license-map.LicenseRef-OtherLicense;
+in license:
+let
+  licenses = spdx.compound-expression license;
+in if licenses == []
+   then otherLicenseWarning license
+   else map (lic: license-map.${lic} or (otherLicenseWarning lic))
+            (pkgs.lib.unique (head licenses)._1)

lib/spdx/licenses.nix

@@ -0,0 +1,20 @@
+pkgs:
+with builtins; let
+  licensesJSON = fromJSON (replaceStrings
+      [ "\\u0026" "\\u0027" "\\u003d" ]
+      [ "&" "'" "=" ]
+      (readFile "${pkgs.evalPackages.spdx-license-list-data}/json/licenses.json")
+    );
+  dropFour = s: substring 0 (stringLength s - 4) s;
+  toSpdx = lic: with lic;
+            { spdxId = licenseId
+            ; shortName = licenseId
+            ; fullName = name
+            ; url = dropFour detailsUrl + "html"
+            ; free = isOsiApproved
+            ;
+            };
+  toNamedValue = lic: { name = lic.spdxId; value = lic; };
+in
+
+listToAttrs (map (l: toNamedValue (toSpdx l)) licensesJSON.licenses)

lib/spdx/parser.nix

@@ -0,0 +1,115 @@
+let String = { mempty = ""; mappend = a: b: a + b; };
+    List = { singleton = x: [x]; };
+    pair = a: b: { _1 = a; _2 = b; };
+    Pair = { pure = s: pair String.mempty s;
+             mappend = a: b: pair (String.mappend a._1 b._1) b._2;
+             map = f: p: pair (f p._1) p._2;
+           };
+    compose = f: g: x: f (g x);
+in with builtins; rec {
+  # Parser a = String -> [(a, String)]
+  string = str: s:
+    let strL = stringLength str;
+        sL = stringLength s;
+    in if str == substring 0 strL s
+       then [ (pair str (substring strL sL s)) ]
+       else [];
+
+  regex = capture_groups: re: s:
+    let result = match "(${re})(.*)" s;
+    in if result == null
+       then []
+       else [ (pair (elemAt result 0) (elemAt result (1 + capture_groups))) ];
+
+  # Left-biased choice
+  choice = cs: s:
+    if cs == []
+    then []
+    else let c = head cs;
+             cs' = tail cs;
+             result = c s;
+         in if result == []
+            then choice cs' s
+            else result;
+
+  optional = p: s:
+    let result = p s;
+    in if result == []
+      then [ (Pair.pure s) ]
+      else result;
+
+  chain = ps: s:
+    if ps == []
+    then [ (Pair.pure s) ]
+    else let p = head ps;
+             ps' = tail ps;
+             result = p s;
+         in if result == []
+            then []
+            else let r = head result;
+                     rest = chain ps' r._2;
+                 in if rest == []
+                    then []
+                    else let r' = head rest;
+                         in [ (Pair.mappend r r') ];
+
+  idstring = regex 0 "[-\.0-9a-zA-Z]+";
+
+  license-ref = let documentref = chain [ (string "DocumentRef-")
+                                          idstring
+                                          (string ":")
+                                        ];
+    in chain [
+      (optional documentref)
+      (string "LicenseRef-")
+      idstring
+    ];
+
+  simple-expression = choice [ license-ref
+                               (chain [ idstring (string "+") ])
+                               idstring
+                             ];
+
+  compound-expression = let wrap = compose (map (Pair.map List.singleton));
+    in choice [
+      (s: let result = simple-expression s;
+          in if result == []
+             then []
+             else let r = head result;
+                      rest = chain [(string " WITH ") idstring ] r._2;
+                  in if rest == []
+                     then []
+                     else [(pair r._1 (head rest)._2)]
+      )
+      (s: let result = simple-expression s;
+          in if result == []
+             then []
+             else let r = head result;
+                      firstLicense = r._1;
+                      operator = choice [ (string " AND ")
+                                          (string " OR ")
+                                        ]
+                                        r._2;
+                  in if operator == []
+                     then []
+                     else let s' = (head operator)._2;
+                              licenses = compound-expression s';
+                          in if licenses == []
+                             then []
+                             else let ls = head licenses;
+                                  in [(pair ([firstLicense] ++ ls._1) ls._2)]
+      )
+      (wrap simple-expression)
+      (s: let openParen = string "(" s;
+          in if openParen == []
+             then []
+             else let result = compound-expression (head openParen)._2;
+                  in if result == []
+                     then []
+                     else let r = head result;
+                              closeParen = string ")" r._2;
+                          in if closeParen == []
+                             then []
+                             else [(pair r._1 (head closeParen)._2)])
+    ];
+}

lib/spdx/test.nix

@@ -0,0 +1,32 @@
+let spdx = import ./parser.nix;
+in {
+  idstringF = [ (spdx.idstring "$@#!$") ];
+  idstringP = [ (spdx.idstring "blah") ];
+
+  license-refF = [ (spdx.license-ref "LicenseRef-$@#!$") ];
+  license-refP = [ (spdx.license-ref "LicenseRef-blah")
+                   (spdx.license-ref "DocumentRef-beep:LicenseRef-boop")
+                 ];
+
+  simple-expressionF = [ (spdx.simple-expression "$@#!$") ];
+  simple-expressionP = [
+    (spdx.simple-expression "blah")
+    (spdx.simple-expression "blah+")
+    (spdx.simple-expression "LicenseRef-blah")
+    (spdx.simple-expression "DocumentRef-beep:LicenseRef-boop")
+  ];
+
+  compound-expressionF = [ (spdx.compound-expression "$@#!$") ];
+  compound-expressionP = [
+    (spdx.compound-expression "blah")
+    (spdx.compound-expression "blah+")
+    (spdx.compound-expression "LicenseRef-blah")
+    (spdx.compound-expression "DocumentRef-beep:LicenseRef-boop")
+    (spdx.compound-expression "(blah)")
+    (spdx.compound-expression "beep OR boop")
+    (spdx.compound-expression "beep AND boop")
+    (spdx.compound-expression "beep WITH boop")
+    (spdx.compound-expression "beep AND (boop OR blap)")
+    (spdx.compound-expression "(beep AND ((boop OR ((blap)))))")
+  ];
+}

modules/configuration-nix.nix

@@ -15,4 +15,11 @@
     pkgs.xorg.libXScrnSaver
     pkgs.xorg.libXinerama
   ];
+
+  # These packages have `license: LGPL` in their .cabal file, but
+  # do not specify the version.  Setting the version here on
+  # examination of the license files included in the packages.
+  packages.hscolour.package.license = pkgs.lib.mkForce "LGPL-2.1-only";
+  packages.cpphs.package.license = pkgs.lib.mkForce "LGPL-2.1-only";
+  packages.polyparse.package.license = pkgs.lib.mkForce "LGPL-2.1-only";
 }