Add/ben tests (#35)

* adding ben tests
* adding ben package and tests
* fixing ben test
* remove commented line

Signed-off-by: vsoch <vsoch@users.noreply.github.com>
Co-authored-by: vsoch <vsoch@users.noreply.github.com>

.gitignore

@@ -1,2 +1,3 @@
 results
 .env
+__pycache__

README.md

@@ -551,7 +551,27 @@ experiment:
 ```
 
 And since the name of the testing file will be mapped to the container name,
-we'd want to call this `libabigail-test-zlib.yaml`.
+we'd want to call this `libabigail-test-zlib.yaml`. If you want to give your
+test a custom Dockerfile base (e.g., some different version of the Dockerfile's
+under [templates](templates) you can add this as an additional section:
+
+```
+packages:
+ # no versions specified implies all
+ - name: ben
+
+tester:
+  name: libabigail
+
+test:
+  dockerfile: templates/ben/Dockerfile
+
+experiment:
+  name: single-test  
+```
+
+The dockerfile should always be in the context of the [templates](templates)
+directory.
  
 ### Reproduce a Test
 

build-si-containers

@@ -48,7 +48,11 @@ package_schema = {
         "package": {
             "type": "object",
             "additionalProperties": False,
-            "required": ["name", "versions", "headers",],
+            "required": [
+                "name",
+                "versions",
+                "headers",
+            ],
             "properties": {
                 "name": {"type": "string"},
                 "versions": {"type": "array", "items": {"type": "string"}},
@@ -62,7 +66,9 @@ package_schema = {
         "test": {
             "type": "object",
             "additionalProperties": False,
-            "properties": {"build_cache": {"type": "boolean"},},
+            "properties": {
+                "build_cache": {"type": "boolean"},
+            },
         },
     },
 }
@@ -109,7 +115,11 @@ test_schema = {
         "test": {
             "type": "object",
             "additionalProperties": False,
-            "properties": {"build_cache": {"type": "boolean"},},
+            "properties": {
+                "prebuilt": {"type": "boolean"},
+                "cache_only": {"type": "boolean"},
+                "dockerfile": {"type": "string"},
+            },
         },
     },
 }
@@ -185,6 +195,17 @@ class Test(Config):
 
     schema = test_schema
 
+    def __init__(self, config_file, **kwargs):
+        # Set and load config file
+        super().__init__(config_file)
+        if "test" not in self.config:
+            self.config["test"] = {}
+
+        # Command line args over-ride config
+        for term in ["prebuilt", "use_cache"]:
+            if term in kwargs:
+                self.config["test"][term] = kwargs.get(term)
+
     def __str__(self):
         return "<test:%s>" % os.path.basename(self.config_file)
 
@@ -335,7 +356,7 @@ class TestSetup:
         """
         # read in this test file
         test_file = self.get_test_config(test)
-        test = Test(test_file)
+        test = Test(test_file, prebuilt=prebuilt, use_cache=use_cache)
 
         # Containers to skip building
         skips = skips or []
@@ -353,13 +374,8 @@ class TestSetup:
 
         tester = test.tester["name"]
 
-        # If we have a test section
-        build_cache = False
-        if test.test:
-            build_cache = test.test.get("build_cache", build_cache)
-
         # Get the tester build template
-        template = self.get_tester_template(tester, build_cache or use_cache, prebuilt)
+        template = self.get_tester_template(tester, test)
         tester = Tester(self.get_tester_config(tester))
 
         # Right now one container has all versions
@@ -465,17 +481,25 @@ class TestSetup:
         template = template.replace(self.templates_dir, "").strip("/")
         return env.get_template(template)
 
-    def get_tester_template(self, tester, use_cache=False, prebuilt=False):
+    def get_tester_template(self, tester, test):
         """
         Given a package and tester, return the container template.
         """
         dockerfile = os.path.join(self.templates_dir, tester, "Dockerfile")
-        if not os.path.exists(dockerfile) and prebuilt:
+        default = os.path.join(self.templates_dir, "Dockerfile.default")
+
+        # Does the test defines a specific dockerfile?
+        test_dockerfile = test.test.get("dockerfile")
+
+        if os.path.exists(test_dockerfile):
+            dockerfile = os.path.abspath(test_dockerfile)
+        elif not os.path.exists(dockerfile) and test.test.get("prebuilt") == True:
             dockerfile = os.path.join(self.templates_dir, "Dockerfile.prebuilt")
-        elif not os.path.exists(dockerfile) and use_cache:
+        elif not os.path.exists(dockerfile) and test.test.get("use_cache" == True):
             dockerfile = os.path.join(self.templates_dir, "Dockerfile.buildcache")
-        else:
-            dockerfile = os.path.join(self.templates_dir, "Dockerfile.default")
+        elif not os.path.exists(dockerfile):
+            dockerfile = default
+
         dockerfile = dockerfile.replace(self.templates_dir, "").strip("/")
         return env.get_template(dockerfile)
 

packages/ben.yaml

@@ -0,0 +1,12 @@
+package:
+ name: ben
+ versions:
+  - 1.0.0
+  - 2.0.0
+ headers:
+  - include
+ libs:
+  - lib/librttiexcep.so
+  - lib/libunderlink.so  
+ bins:
+  - bin/underlinktest

spack/packages/ben/package.py

@@ -0,0 +1,23 @@
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class Ben(MakefilePackage):
+    homepage = "https://github.com/buildsi/build-abi-test-ben"
+    url = "https://github.com/buildsi/build-abi-test-ben/archive/refs/tags/2.0.0.tar.gz"
+    git = "https://github.com/buildsi/build-abi-test-ben.git"
+
+    version('main', branch='main')
+    version('1.0.0', sha256='28d85b076c7127934b1127c1695bbdc16e59ec5a42def68478c938835fbe114f')
+    version('2.0.0', sha256='d913cf8de6ed92c836388bc7a492e0f975e589bab7616d99607180559d487d72')
+
+    def install(self, spec, prefix):
+        mkdir(prefix.bin)
+        mkdir(prefix.lib)
+        install('underlinktest', prefix.bin)
+        install('librttiexcep.so', prefix.lib)
+        install('libunderlink.so', prefix.lib)

templates/Dockerfile.default

@@ -9,7 +9,7 @@ COPY spack/ /test-packages
 RUN apt-get update && apt-get install -y curl python3-botocore python3-boto3 && \
     spack repo add /test-packages && \
     spack config add -f "packages.yaml" && \
-    {% for package in packages %}{% for version in package.versions %}spack install --source {% if cache_only %}--cache-only{% endif %} --deprecated {{ package.name }}@{{ version }} && \{% endfor %}{% endfor %}
+    {% for package in packages %}{% for version in package.versions %}spack install --no-checksum --source {% if cache_only %}--cache-only{% endif %} --deprecated {{ package.name }}@{{ version }} && \{% endfor %}{% endfor %}
     printf "Finished installation attempts\n"
 
 FROM ghcr.io/buildsi/{{ tester.name }}:{{ tester.version }}

templates/ben/Dockerfile

@@ -0,0 +1,29 @@
+FROM gcc as base
+ENV DEBIAN_FRONTEND=noninteractive
+WORKDIR /code
+
+RUN apt-get update && apt-get install -y git curl python3-botocore python3-boto3
+RUN git clone --depth 1 https://github.com/spack/spack /opt/spack
+ENV PATH=/opt/spack/bin:$PATH
+RUN echo "packages:" > packages.yaml &&\
+    echo "  all:" >> packages.yaml &&\
+    echo "    target: [x86_64]" >> packages.yaml
+
+COPY spack/ /test-packages
+RUN spack repo add /test-packages && \
+    spack config add -f "packages.yaml" && \
+    {% for package in packages %}{% for version in package.versions %}spack install --no-checksum --source {% if cache_only %}--cache-only{% endif %} --deprecated {{ package.name }}@{{ version }} && \{% endfor %}{% endfor %}
+    printf "Finished installation attempts\n"
+
+FROM ghcr.io/buildsi/{{ tester.name }}:{{ tester.version }}
+COPY --from=base /opt/spack /opt/spack
+
+WORKDIR /build-si/
+ENV PATH=/opt/spack/bin:$PATH
+COPY {{ test.config_basename }} /build-si/tests.yaml
+COPY {{ tester.runscript }} /build-si/{{ tester.runscript }}
+{% for bin in bins %}COPY {{ bin }} /usr/local/bin/{{ bin }}
+{% endfor %}
+RUN apt-get install -y time && \
+    mkdir -p /results && chmod +x /build-si/{{ tester.runscript }} {% if bins %}{% for bin in bins %} && chmod +x /usr/local/bin/{{ bin }}{% endfor %}{% endif %}
+ENTRYPOINT ["{{ tester.entrypoint }}", {% for arg in tester.args %}"{{ arg }}", {% endfor %}"/build-si/{{ tester.runscript }}"]

tests/libabigail-test-ben.yaml

@@ -0,0 +1,12 @@
+packages:
+ # no versions specified implies all
+ - name: ben
+
+tester:
+  name: libabigail
+
+test:
+  dockerfile: templates/ben/Dockerfile
+
+experiment:
+  name: single-test  

tests/smeagle-test-ben.yaml

@@ -0,0 +1,13 @@
+packages:
+ # no versions specified implies all
+ - name: ben
+
+tester:
+  name: smeagle
+
+test:
+  dockerfile: templates/ben/Dockerfile
+
+experiment:
+  name: single-test
+