Specify branch and directory for github reference files (#56)

* Checkout git ref for reference files repository

  - New referenceFiles.git-ref setting to specify a git
    reference, e.g. a branch name, tag or commit hash.

* Test that PNlib v2.2 has 93/90 simulating/verifying models

.github/workflows/test.yml

@@ -55,6 +55,15 @@ jobs:
         run: |
           python .github/scripts/archiveResults.py "PNlib" "2.2.0" "${{ matrix.omc-version }}" "html/"
 
+      - name: Verify that overview.html contains simulation and verification
+        shell: bash
+        run: |
+          if ! grep -q "<tr><td><a href=\"${{ matrix.omc-version }}/PNlib_2.2.0/PNlib_2.2.0.html\">${{ matrix.omc-version }}</a></td><td><a>93</a></td><td><a>93</a></td><td><a>92</a></td><td><a>92</a></td><td><a>92</a></td><td><a>92</a></td><td><a>92</a></td><td><a>91</a></td><td><a>90</a></td></tr></table>" overview.html; then
+            echo "Failed to find 93 simulating and 90 verifying models in overwiew.html:"
+            cat overview.html
+            exit 1
+          fi
+
       - name: Archive sqlite3.db
         uses: actions/upload-artifact@v4
         if: always()

.gitignore

@@ -1,11 +1,13 @@
 __pycache__
 !HelloWorld.mo
 !HelloWorld.mos
+/*.files
+/*.html
 /*.json
 /converted-libraries
+/PNlib*/
+/ReferenceFiles/
 files/
 HelloWorld*
 history/
 sqlite3.db
-/*.html
-/*.files

configs/sanityCheck.json

@@ -1,6 +1,14 @@
 [
   {
     "library": "PNlib",
-    "libraryVersion": "2.2.0"
+    "libraryVersion": "2.2.0",
+    "referenceFileExtension":"mat",
+    "referenceFileNameDelimiter":".",
+    "referenceFiles":{
+      "giturl":"https://github.com/AMIT-HSBI/PNlib",
+      "destination":"ReferenceFiles/PNlib",
+      "git-ref": "v2.2",
+      "git-directory": "ReferenceFiles"
+    }
   }
 ]

test.py

@@ -345,25 +345,47 @@ def testHelloWorld(cmd):
           raise Exception("Environment variable %s not defined, but used in JSON config for reference files" % k)
         c["referenceFiles"] = c["referenceFiles"].replace(m.group(0), os.environ[k])
     elif "giturl" in c["referenceFiles"]:
+      refFilesGitTag = "origin/master"
+      if "git-ref" in c["referenceFiles"]:
+        refFilesGitTag = c["referenceFiles"]["git-ref"].strip()
       if c["referenceFiles"]["destination"] in preparedReferenceDirs:
         (c["referenceFiles"],c["referenceFilesURL"]) = preparedReferenceDirs[destination]
         continue
       giturl = c["referenceFiles"]["giturl"]
       destination = c["referenceFiles"]["destination"]
-      if not os.path.isdir(destination):
-        subprocess.check_call(["git", "clone", giturl, destination], stderr=subprocess.STDOUT)
       destinationReal = os.path.realpath(destination)
+
+      if not os.path.isdir(destination):
+        if "git-directory" in c["referenceFiles"]:
+          # Sparse clone
+          os.makedirs(destination)
+          subprocess.check_call(["git", "init"], stderr=subprocess.STDOUT, cwd=destinationReal)
+          subprocess.check_call(["git", "remote", "add", "-f", "origin", giturl], stderr=subprocess.STDOUT, cwd=destinationReal)
+          subprocess.check_call(["git", "config", "core.sparseCheckout", "true"], stderr=subprocess.STDOUT, cwd=destinationReal)
+          file = open(os.path.join(destinationReal,".git", "info", "sparse-checkout"), "a")
+          file.write(c["referenceFiles"]["git-directory"].strip())
+          file.close()
+        else:
+          # Clone
+          subprocess.check_call(["git", "clone", giturl, destination], stderr=subprocess.STDOUT)
+
       subprocess.check_call(["git", "clean", "-fdx", "--exclude=*.hash"], stderr=subprocess.STDOUT, cwd=destinationReal)
       subprocess.check_call(["git", "fetch", "origin"], stderr=subprocess.STDOUT, cwd=destination)
-      subprocess.check_call(["git", "reset", "--hard", "origin/master"], stderr=subprocess.STDOUT, cwd=destinationReal)
+      subprocess.check_call(["git", "reset", "--hard", refFilesGitTag], stderr=subprocess.STDOUT, cwd=destinationReal)
       subprocess.check_call(["git", "clean", "-fdx", "--exclude=*.hash"], stderr=subprocess.STDOUT, cwd=destinationReal)
       subprocess.check_call(["find", ".", "-name", "*.mat.xz", "-exec", "xz", "--decompress", "--keep", "{}", ";"], stderr=subprocess.STDOUT, cwd=destinationReal)
-      githash = subprocess.check_output(["git", "rev-parse", "--verify", "HEAD"], stderr=subprocess.STDOUT, cwd=destinationReal)
-      c["referenceFiles"] = destinationReal
+      githash = subprocess.check_output(["git", "rev-parse", "--verify", "HEAD"], stderr=subprocess.STDOUT, cwd=destinationReal, encoding='utf8')
+
+      if "git-directory" in c["referenceFiles"]:
+        c["referenceFiles"] = os.path.join(destinationReal, c["referenceFiles"]['git-directory'])
+        print(c["referenceFiles"])
+      else:
+        c["referenceFiles"] = destinationReal
+
       if giturl.startswith("https://github.com"):
-        c["referenceFilesURL"] = '<a href="%s/tree/%s">%s (%s)</a>' % (giturl,githash.strip(),giturl,githash.strip())
+        c["referenceFilesURL"] = '<a href="%s/tree/%s">%s (%s)</a>' % (giturl, githash.strip(), giturl, githash.strip())
       else:
-        c["referenceFilesURL"] = "%s (%s)" % (giturl,githash.strip())
+        c["referenceFilesURL"] = "%s (%s)" % (giturl, githash.strip())
       preparedReferenceDirs[destination] = (c["referenceFiles"],c["referenceFilesURL"])
     else:
       raise Exception("Unknown referenceFiles in config: %s" % (str(c["referenceFiles"])))