diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..8b9f53606 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +nix/** linguist-generated diff --git a/CHANGELOG.md b/CHANGELOG.md index 91f91a6e4..50d3c0af1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,8 +27,9 @@ All notable changes to this project will be documented in this file. - ci: Rename local actions, adjust action inputs and outputs, add definition README file ([#819]). -- Update cargo-cyclonedx to 0.5.5 and build CycloneDX 1.5 files ([#783]) -- Enable [Docker build checks](https://docs.docker.com/build/checks/) ([#872]) +- Update cargo-cyclonedx to 0.5.5 and build CycloneDX 1.5 files ([#783]). +- Enable [Docker build checks](https://docs.docker.com/build/checks/) ([#872]). +- java: migrate to temurin jdk/jre ([#894]). ### Removed @@ -78,6 +79,7 @@ All notable changes to this project will be documented in this file. [#880]: https://github.com/stackabletech/docker-images/pull/880 [#881]: https://github.com/stackabletech/docker-images/pull/881 [#882]: https://github.com/stackabletech/docker-images/pull/882 +[#894]: https://github.com/stackabletech/docker-images/pull/894 ## [24.7.0] - 2024-07-24 diff --git a/hive/versions.py b/hive/versions.py index fb8e34644..45dbf096f 100644 --- a/hive/versions.py +++ b/hive/versions.py @@ -4,7 +4,7 @@ "jmx_exporter": "1.0.1", # Hive 4 must be built with Java 8 (according to GitHub README) but seems to run on Java 11 "java-base": "11", - "java-devel": "1.8.0", + "java-devel": "8", "hadoop": "3.3.6", # Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 "aws_java_sdk_bundle": "1.12.367", @@ -16,7 +16,7 @@ "jmx_exporter": "1.0.1", # Hive 3 must be built with Java 8 but will run on Java 11 "java-base": "11", - "java-devel": "1.8.0", + "java-devel": "8", "hadoop": "3.3.6", # Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 "aws_java_sdk_bundle": "1.12.367", diff --git a/java-base/Dockerfile b/java-base/Dockerfile index 74f2597cd..c752b415b 100644 --- a/java-base/Dockerfile +++ b/java-base/Dockerfile @@ -17,13 +17,20 @@ LABEL name="Stackable image for OpenJDK" \ summary="The Stackable OpenJDK base image." \ description="This image is the base image for all Stackable Java product images." -# We need to use EPEL, as openjdk 22 is not shipped with UBI9 -RUN rpm --install --replacepkgs https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm +# See: https://adoptium.net/en-gb/installation/linux/#_centosrhelfedora_instructions +RUN cat < /etc/yum.repos.d/adoptium.repo +[Adoptium] +name=Adoptium +baseurl=https://packages.adoptium.net/artifactory/rpm/rhel/\$releasever/\$basearch +enabled=1 +gpgcheck=1 +gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public +EOF RUN microdnf update && \ microdnf install \ - # Obviously needed to run Java programs - java-${PRODUCT}-openjdk-headless \ + # Needed to run Java programs + "temurin-${PRODUCT}-jre" \ # Needed, because otherwise e.g. Zookeeper fails with # Caused by: java.io.FileNotFoundException: /usr/lib/jvm/java-11-openjdk-11.0.20.0.8-2.el8.x86_64/lib/tzdb.dat (No such file or directory) tzdata-java \ @@ -34,7 +41,7 @@ RUN microdnf update && \ COPY java-base/licenses /licenses -ENV JAVA_HOME=/usr/lib/jvm/jre-${PRODUCT} +ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT}-jre" # This image doesn't include the development packages for Java. # For images that need the devel package (ex. Spark) use this env variable to diff --git a/java-base/versions.py b/java-base/versions.py index 46b6e9fd9..37b62eea1 100644 --- a/java-base/versions.py +++ b/java-base/versions.py @@ -1,6 +1,6 @@ versions = [ { - "product": "1.8.0", + "product": "8", "vector": "0.41.1", }, { diff --git a/java-devel/Dockerfile b/java-devel/Dockerfile index 4e4b9cb98..03223068f 100644 --- a/java-devel/Dockerfile +++ b/java-devel/Dockerfile @@ -9,8 +9,15 @@ FROM stackable/image/stackable-base ARG PRODUCT -# We need to use EPEL, as openjdk 22 is not shipped with UBI9 -RUN rpm --install --replacepkgs https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm +# See: https://adoptium.net/en-gb/installation/linux/#_centosrhelfedora_instructions +RUN cat < /etc/yum.repos.d/adoptium.repo +[Adoptium] +name=Adoptium +baseurl=https://packages.adoptium.net/artifactory/rpm/rhel/\$releasever/\$basearch +enabled=1 +gpgcheck=1 +gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public +EOF RUN microdnf update && \ microdnf install -y \ @@ -27,9 +34,10 @@ RUN microdnf update && \ gettext \ # For the apply_patches.sh script git \ - # needed by the maven ant run plugin for the "set-hostname-property" step in zookeeper + # Needed by the maven ant run plugin for the "set-hostname-property" step in zookeeper hostname \ - java-"${PRODUCT}"-openjdk-devel \ + # Needed for compiling Java projects + "temurin-${PRODUCT}-jdk" \ krb5-devel \ libcurl-devel \ make \ @@ -46,7 +54,7 @@ RUN microdnf update && \ microdnf clean all && \ rm -rf /var/cache/yum -ENV JAVA_HOME=/usr/lib/jvm/jre-${PRODUCT} +ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT}-jdk" COPY --chown=stackable:0 java-devel/stackable/settings.xml /stackable/.m2/settings.xml diff --git a/java-devel/versions.py b/java-devel/versions.py index 4438c47d7..6503d6466 100644 --- a/java-devel/versions.py +++ b/java-devel/versions.py @@ -1,6 +1,6 @@ versions = [ { - "product": "1.8.0", + "product": "8", "stackable-base": "1.0.0", }, { diff --git a/nix/sources.json b/nix/sources.json new file mode 100644 index 000000000..f9a6f291f --- /dev/null +++ b/nix/sources.json @@ -0,0 +1,26 @@ +{ + "image-tools": { + "branch": "main", + "description": "Tools to manipulate container images of the Stackable Data Platform", + "homepage": null, + "owner": "stackabletech", + "repo": "image-tools", + "rev": "104d3ad24eca5399977e9ae97b64235abac9c7c0", + "sha256": "1dg66jrsxam9m8z8v9fyr00hsd3rspma5lad9s8kbnyj71rq0rrw", + "type": "tarball", + "url": "https://github.com/stackabletech/image-tools/archive/104d3ad24eca5399977e9ae97b64235abac9c7c0.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs": { + "branch": "nixos-unstable", + "description": "Nix Packages collection", + "homepage": null, + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "6c43a3495a11e261e5f41e5d7eda2d71dae1b2fe", + "sha256": "16f329z831bq7l3wn1dfvbkh95l2gcggdwn6rk3cisdmv2aa3189", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/6c43a3495a11e261e5f41e5d7eda2d71dae1b2fe.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + } +} diff --git a/nix/sources.nix b/nix/sources.nix new file mode 100644 index 000000000..fe3dadf7e --- /dev/null +++ b/nix/sources.nix @@ -0,0 +1,198 @@ +# This file has been generated by Niv. + +let + + # + # The fetchers. fetch_ fetches specs of type . + # + + fetch_file = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchurl { inherit (spec) url sha256; name = name'; } + else + pkgs.fetchurl { inherit (spec) url sha256; name = name'; }; + + fetch_tarball = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchTarball { name = name'; inherit (spec) url sha256; } + else + pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; + + fetch_git = name: spec: + let + ref = + spec.ref or ( + if spec ? branch then "refs/heads/${spec.branch}" else + if spec ? tag then "refs/tags/${spec.tag}" else + abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!" + ); + submodules = spec.submodules or false; + submoduleArg = + let + nixSupportsSubmodules = builtins.compareVersions builtins.nixVersion "2.4" >= 0; + emptyArgWithWarning = + if submodules + then + builtins.trace + ( + "The niv input \"${name}\" uses submodules " + + "but your nix's (${builtins.nixVersion}) builtins.fetchGit " + + "does not support them" + ) + { } + else { }; + in + if nixSupportsSubmodules + then { inherit submodules; } + else emptyArgWithWarning; + in + builtins.fetchGit + ({ url = spec.repo; inherit (spec) rev; inherit ref; } // submoduleArg); + + fetch_local = spec: spec.path; + + fetch_builtin-tarball = name: throw + ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=tarball -a builtin=true''; + + fetch_builtin-url = name: throw + ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=file -a builtin=true''; + + # + # Various helpers + # + + # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 + sanitizeName = name: + ( + concatMapStrings (s: if builtins.isList s then "-" else s) + ( + builtins.split "[^[:alnum:]+._?=-]+" + ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) + ) + ); + + # The set of packages used when specs are fetched using non-builtins. + mkPkgs = sources: system: + let + sourcesNixpkgs = + import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; }; + hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; + hasThisAsNixpkgsPath = == ./.; + in + if builtins.hasAttr "nixpkgs" sources + then sourcesNixpkgs + else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then + import { } + else + abort + '' + Please specify either (through -I or NIX_PATH=nixpkgs=...) or + add a package called "nixpkgs" to your sources.json. + ''; + + # The actual fetching function. + fetch = pkgs: name: spec: + + if ! builtins.hasAttr "type" spec then + abort "ERROR: niv spec ${name} does not have a 'type' attribute" + else if spec.type == "file" then fetch_file pkgs name spec + else if spec.type == "tarball" then fetch_tarball pkgs name spec + else if spec.type == "git" then fetch_git name spec + else if spec.type == "local" then fetch_local spec + else if spec.type == "builtin-tarball" then fetch_builtin-tarball name + else if spec.type == "builtin-url" then fetch_builtin-url name + else + abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; + + # If the environment variable NIV_OVERRIDE_${name} is set, then use + # the path directly as opposed to the fetched source. + replace = name: drv: + let + saneName = stringAsChars (c: if (builtins.match "[a-zA-Z0-9]" c) == null then "_" else c) name; + ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; + in + if ersatz == "" then drv else + # this turns the string into an actual Nix path (for both absolute and + # relative paths) + if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}"; + + # Ports of functions for older nix versions + + # a Nix version of mapAttrs if the built-in doesn't exist + mapAttrs = builtins.mapAttrs or ( + f: set: with builtins; + listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) + ); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 + range = first: last: if first > last then [ ] else builtins.genList (n: first + n) (last - first + 1); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 + stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 + stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); + concatMapStrings = f: list: concatStrings (map f list); + concatStrings = builtins.concatStringsSep ""; + + # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331 + optionalAttrs = cond: as: if cond then as else { }; + + # fetchTarball version that is compatible between all the versions of Nix + builtins_fetchTarball = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchTarball; + in + if lessThan nixVersion "1.12" then + fetchTarball ({ inherit url; } // (optionalAttrs (name != null) { inherit name; })) + else + fetchTarball attrs; + + # fetchurl version that is compatible between all the versions of Nix + builtins_fetchurl = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchurl; + in + if lessThan nixVersion "1.12" then + fetchurl ({ inherit url; } // (optionalAttrs (name != null) { inherit name; })) + else + fetchurl attrs; + + # Create the final "sources" from the config + mkSources = config: + mapAttrs + ( + name: spec: + if builtins.hasAttr "outPath" spec + then + abort + "The values in sources.json should not have an 'outPath' attribute" + else + spec // { outPath = replace name (fetch config.pkgs name spec); } + ) + config.sources; + + # The "config" used by the fetchers + mkConfig = + { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null + , sources ? if sourcesFile == null then { } else builtins.fromJSON (builtins.readFile sourcesFile) + , system ? builtins.currentSystem + , pkgs ? mkPkgs sources system + }: rec { + # The sources, i.e. the attribute set of spec name to spec + inherit sources; + + # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers + inherit pkgs; + }; + +in +mkSources (mkConfig { }) // { __functor = _: settings: mkSources (mkConfig settings); } diff --git a/opa/Dockerfile b/opa/Dockerfile index 6420ad5f2..31aa3f130 100644 --- a/opa/Dockerfile +++ b/opa/Dockerfile @@ -9,7 +9,6 @@ ARG BUNDLE_BUILDER_VERSION RUN microdnf update \ && microdnf install \ cmake \ - findutils \ gcc \ gcc-c++ \ git \ diff --git a/shell.nix b/shell.nix new file mode 100644 index 000000000..09a26e528 --- /dev/null +++ b/shell.nix @@ -0,0 +1,13 @@ +{ sources ? import ./nix/sources.nix +, nixpkgs ? sources.nixpkgs +, pkgs ? import nixpkgs { } +}: + +let + bake = pkgs.callPackage (sources.image-tools + "/image-tools.nix") { }; +in +pkgs.mkShell { + packages = with pkgs; [ + bake + ]; +} diff --git a/stackable-base/Dockerfile b/stackable-base/Dockerfile index 3833d8d35..b7184dcc3 100644 --- a/stackable-base/Dockerfile +++ b/stackable-base/Dockerfile @@ -80,6 +80,12 @@ COPY stackable-base/stackable/curlrc /root/.curlrc RUN <