From f5e8fd44a81a4355ced41d9fc4883829566786ed Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 14 Oct 2024 12:01:13 +0200 Subject: [PATCH 1/8] chore: add shell.nix --- hive/versions.py | 4 +- nix/sources.json | 26 +++++++ nix/sources.nix | 198 +++++++++++++++++++++++++++++++++++++++++++++++ shell.nix | 13 ++++ 4 files changed, 239 insertions(+), 2 deletions(-) create mode 100644 nix/sources.json create mode 100644 nix/sources.nix create mode 100644 shell.nix diff --git a/hive/versions.py b/hive/versions.py index fb8e34644..45dbf096f 100644 --- a/hive/versions.py +++ b/hive/versions.py @@ -4,7 +4,7 @@ "jmx_exporter": "1.0.1", # Hive 4 must be built with Java 8 (according to GitHub README) but seems to run on Java 11 "java-base": "11", - "java-devel": "1.8.0", + "java-devel": "8", "hadoop": "3.3.6", # Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 "aws_java_sdk_bundle": "1.12.367", @@ -16,7 +16,7 @@ "jmx_exporter": "1.0.1", # Hive 3 must be built with Java 8 but will run on Java 11 "java-base": "11", - "java-devel": "1.8.0", + "java-devel": "8", "hadoop": "3.3.6", # Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 "aws_java_sdk_bundle": "1.12.367", diff --git a/nix/sources.json b/nix/sources.json new file mode 100644 index 000000000..f9a6f291f --- /dev/null +++ b/nix/sources.json @@ -0,0 +1,26 @@ +{ + "image-tools": { + "branch": "main", + "description": "Tools to manipulate container images of the Stackable Data Platform", + "homepage": null, + "owner": "stackabletech", + "repo": "image-tools", + "rev": "104d3ad24eca5399977e9ae97b64235abac9c7c0", + "sha256": "1dg66jrsxam9m8z8v9fyr00hsd3rspma5lad9s8kbnyj71rq0rrw", + "type": "tarball", + "url": "https://github.com/stackabletech/image-tools/archive/104d3ad24eca5399977e9ae97b64235abac9c7c0.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs": { + "branch": "nixos-unstable", + "description": "Nix Packages collection", + "homepage": null, + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "6c43a3495a11e261e5f41e5d7eda2d71dae1b2fe", + "sha256": "16f329z831bq7l3wn1dfvbkh95l2gcggdwn6rk3cisdmv2aa3189", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/6c43a3495a11e261e5f41e5d7eda2d71dae1b2fe.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + } +} diff --git a/nix/sources.nix b/nix/sources.nix new file mode 100644 index 000000000..fe3dadf7e --- /dev/null +++ b/nix/sources.nix @@ -0,0 +1,198 @@ +# This file has been generated by Niv. + +let + + # + # The fetchers. fetch_ fetches specs of type . + # + + fetch_file = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchurl { inherit (spec) url sha256; name = name'; } + else + pkgs.fetchurl { inherit (spec) url sha256; name = name'; }; + + fetch_tarball = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchTarball { name = name'; inherit (spec) url sha256; } + else + pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; + + fetch_git = name: spec: + let + ref = + spec.ref or ( + if spec ? branch then "refs/heads/${spec.branch}" else + if spec ? tag then "refs/tags/${spec.tag}" else + abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!" + ); + submodules = spec.submodules or false; + submoduleArg = + let + nixSupportsSubmodules = builtins.compareVersions builtins.nixVersion "2.4" >= 0; + emptyArgWithWarning = + if submodules + then + builtins.trace + ( + "The niv input \"${name}\" uses submodules " + + "but your nix's (${builtins.nixVersion}) builtins.fetchGit " + + "does not support them" + ) + { } + else { }; + in + if nixSupportsSubmodules + then { inherit submodules; } + else emptyArgWithWarning; + in + builtins.fetchGit + ({ url = spec.repo; inherit (spec) rev; inherit ref; } // submoduleArg); + + fetch_local = spec: spec.path; + + fetch_builtin-tarball = name: throw + ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=tarball -a builtin=true''; + + fetch_builtin-url = name: throw + ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=file -a builtin=true''; + + # + # Various helpers + # + + # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 + sanitizeName = name: + ( + concatMapStrings (s: if builtins.isList s then "-" else s) + ( + builtins.split "[^[:alnum:]+._?=-]+" + ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) + ) + ); + + # The set of packages used when specs are fetched using non-builtins. + mkPkgs = sources: system: + let + sourcesNixpkgs = + import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; }; + hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; + hasThisAsNixpkgsPath = == ./.; + in + if builtins.hasAttr "nixpkgs" sources + then sourcesNixpkgs + else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then + import { } + else + abort + '' + Please specify either (through -I or NIX_PATH=nixpkgs=...) or + add a package called "nixpkgs" to your sources.json. + ''; + + # The actual fetching function. + fetch = pkgs: name: spec: + + if ! builtins.hasAttr "type" spec then + abort "ERROR: niv spec ${name} does not have a 'type' attribute" + else if spec.type == "file" then fetch_file pkgs name spec + else if spec.type == "tarball" then fetch_tarball pkgs name spec + else if spec.type == "git" then fetch_git name spec + else if spec.type == "local" then fetch_local spec + else if spec.type == "builtin-tarball" then fetch_builtin-tarball name + else if spec.type == "builtin-url" then fetch_builtin-url name + else + abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; + + # If the environment variable NIV_OVERRIDE_${name} is set, then use + # the path directly as opposed to the fetched source. + replace = name: drv: + let + saneName = stringAsChars (c: if (builtins.match "[a-zA-Z0-9]" c) == null then "_" else c) name; + ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; + in + if ersatz == "" then drv else + # this turns the string into an actual Nix path (for both absolute and + # relative paths) + if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}"; + + # Ports of functions for older nix versions + + # a Nix version of mapAttrs if the built-in doesn't exist + mapAttrs = builtins.mapAttrs or ( + f: set: with builtins; + listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) + ); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 + range = first: last: if first > last then [ ] else builtins.genList (n: first + n) (last - first + 1); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 + stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 + stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); + concatMapStrings = f: list: concatStrings (map f list); + concatStrings = builtins.concatStringsSep ""; + + # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331 + optionalAttrs = cond: as: if cond then as else { }; + + # fetchTarball version that is compatible between all the versions of Nix + builtins_fetchTarball = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchTarball; + in + if lessThan nixVersion "1.12" then + fetchTarball ({ inherit url; } // (optionalAttrs (name != null) { inherit name; })) + else + fetchTarball attrs; + + # fetchurl version that is compatible between all the versions of Nix + builtins_fetchurl = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchurl; + in + if lessThan nixVersion "1.12" then + fetchurl ({ inherit url; } // (optionalAttrs (name != null) { inherit name; })) + else + fetchurl attrs; + + # Create the final "sources" from the config + mkSources = config: + mapAttrs + ( + name: spec: + if builtins.hasAttr "outPath" spec + then + abort + "The values in sources.json should not have an 'outPath' attribute" + else + spec // { outPath = replace name (fetch config.pkgs name spec); } + ) + config.sources; + + # The "config" used by the fetchers + mkConfig = + { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null + , sources ? if sourcesFile == null then { } else builtins.fromJSON (builtins.readFile sourcesFile) + , system ? builtins.currentSystem + , pkgs ? mkPkgs sources system + }: rec { + # The sources, i.e. the attribute set of spec name to spec + inherit sources; + + # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers + inherit pkgs; + }; + +in +mkSources (mkConfig { }) // { __functor = _: settings: mkSources (mkConfig settings); } diff --git a/shell.nix b/shell.nix new file mode 100644 index 000000000..09a26e528 --- /dev/null +++ b/shell.nix @@ -0,0 +1,13 @@ +{ sources ? import ./nix/sources.nix +, nixpkgs ? sources.nixpkgs +, pkgs ? import nixpkgs { } +}: + +let + bake = pkgs.callPackage (sources.image-tools + "/image-tools.nix") { }; +in +pkgs.mkShell { + packages = with pkgs; [ + bake + ]; +} From 6b0af74b7c53759c8d681045eb82dc4d72234c22 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 14 Oct 2024 13:02:57 +0200 Subject: [PATCH 2/8] chore: install find in stackable-base --- opa/Dockerfile | 1 - stackable-base/Dockerfile | 7 +++++++ superset/Dockerfile | 2 -- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/opa/Dockerfile b/opa/Dockerfile index 6420ad5f2..31aa3f130 100644 --- a/opa/Dockerfile +++ b/opa/Dockerfile @@ -9,7 +9,6 @@ ARG BUNDLE_BUILDER_VERSION RUN microdnf update \ && microdnf install \ cmake \ - findutils \ gcc \ gcc-c++ \ git \ diff --git a/stackable-base/Dockerfile b/stackable-base/Dockerfile index 3833d8d35..84989120a 100644 --- a/stackable-base/Dockerfile +++ b/stackable-base/Dockerfile @@ -80,6 +80,12 @@ COPY stackable-base/stackable/curlrc /root/.curlrc RUN < Date: Mon, 14 Oct 2024 13:34:29 +0200 Subject: [PATCH 3/8] chore: migrate to temurin jdk/jre --- java-base/Dockerfile | 16 +++++++++++----- java-base/versions.py | 2 +- java-devel/Dockerfile | 17 ++++++++++++----- java-devel/versions.py | 2 +- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/java-base/Dockerfile b/java-base/Dockerfile index 74f2597cd..9966e0611 100644 --- a/java-base/Dockerfile +++ b/java-base/Dockerfile @@ -17,13 +17,19 @@ LABEL name="Stackable image for OpenJDK" \ summary="The Stackable OpenJDK base image." \ description="This image is the base image for all Stackable Java product images." -# We need to use EPEL, as openjdk 22 is not shipped with UBI9 -RUN rpm --install --replacepkgs https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm +RUN cat < /etc/yum.repos.d/adoptium.repo +[Adoptium] +name=Adoptium +baseurl=https://packages.adoptium.net/artifactory/rpm/rhel/\$releasever/\$basearch +enabled=1 +gpgcheck=1 +gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public +EOF RUN microdnf update && \ microdnf install \ - # Obviously needed to run Java programs - java-${PRODUCT}-openjdk-headless \ + # Needed to run Java programs + "temurin-${PRODUCT}-jre" \ # Needed, because otherwise e.g. Zookeeper fails with # Caused by: java.io.FileNotFoundException: /usr/lib/jvm/java-11-openjdk-11.0.20.0.8-2.el8.x86_64/lib/tzdb.dat (No such file or directory) tzdata-java \ @@ -34,7 +40,7 @@ RUN microdnf update && \ COPY java-base/licenses /licenses -ENV JAVA_HOME=/usr/lib/jvm/jre-${PRODUCT} +ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT}-jre" # This image doesn't include the development packages for Java. # For images that need the devel package (ex. Spark) use this env variable to diff --git a/java-base/versions.py b/java-base/versions.py index 46b6e9fd9..37b62eea1 100644 --- a/java-base/versions.py +++ b/java-base/versions.py @@ -1,6 +1,6 @@ versions = [ { - "product": "1.8.0", + "product": "8", "vector": "0.41.1", }, { diff --git a/java-devel/Dockerfile b/java-devel/Dockerfile index 4e4b9cb98..304165b60 100644 --- a/java-devel/Dockerfile +++ b/java-devel/Dockerfile @@ -9,8 +9,14 @@ FROM stackable/image/stackable-base ARG PRODUCT -# We need to use EPEL, as openjdk 22 is not shipped with UBI9 -RUN rpm --install --replacepkgs https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm +RUN cat < /etc/yum.repos.d/adoptium.repo +[Adoptium] +name=Adoptium +baseurl=https://packages.adoptium.net/artifactory/rpm/rhel/\$releasever/\$basearch +enabled=1 +gpgcheck=1 +gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public +EOF RUN microdnf update && \ microdnf install -y \ @@ -27,9 +33,10 @@ RUN microdnf update && \ gettext \ # For the apply_patches.sh script git \ - # needed by the maven ant run plugin for the "set-hostname-property" step in zookeeper + # Needed by the maven ant run plugin for the "set-hostname-property" step in zookeeper hostname \ - java-"${PRODUCT}"-openjdk-devel \ + # Needed for compiling Java projects + "temurin-${PRODUCT}-jdk" \ krb5-devel \ libcurl-devel \ make \ @@ -46,7 +53,7 @@ RUN microdnf update && \ microdnf clean all && \ rm -rf /var/cache/yum -ENV JAVA_HOME=/usr/lib/jvm/jre-${PRODUCT} +ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT}-jdk" COPY --chown=stackable:0 java-devel/stackable/settings.xml /stackable/.m2/settings.xml diff --git a/java-devel/versions.py b/java-devel/versions.py index 4438c47d7..6503d6466 100644 --- a/java-devel/versions.py +++ b/java-devel/versions.py @@ -1,6 +1,6 @@ versions = [ { - "product": "1.8.0", + "product": "8", "stackable-base": "1.0.0", }, { From 1f9843bb796b10b31fe2c8bac3986625d4e84750 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 14 Oct 2024 13:41:58 +0200 Subject: [PATCH 4/8] chore: mark niv files as generated --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..8b9f53606 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +nix/** linguist-generated From 7a773c208bb26df72872fa33f6819a7c67be2d29 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 14 Oct 2024 13:51:10 +0200 Subject: [PATCH 5/8] chore: update the changelog --- CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91f91a6e4..50d3c0af1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,8 +27,9 @@ All notable changes to this project will be documented in this file. - ci: Rename local actions, adjust action inputs and outputs, add definition README file ([#819]). -- Update cargo-cyclonedx to 0.5.5 and build CycloneDX 1.5 files ([#783]) -- Enable [Docker build checks](https://docs.docker.com/build/checks/) ([#872]) +- Update cargo-cyclonedx to 0.5.5 and build CycloneDX 1.5 files ([#783]). +- Enable [Docker build checks](https://docs.docker.com/build/checks/) ([#872]). +- java: migrate to temurin jdk/jre ([#894]). ### Removed @@ -78,6 +79,7 @@ All notable changes to this project will be documented in this file. [#880]: https://github.com/stackabletech/docker-images/pull/880 [#881]: https://github.com/stackabletech/docker-images/pull/881 [#882]: https://github.com/stackabletech/docker-images/pull/882 +[#894]: https://github.com/stackabletech/docker-images/pull/894 ## [24.7.0] - 2024-07-24 From 577852f61416d87ca7386b499e4bf4e28ec0f065 Mon Sep 17 00:00:00 2001 From: Techassi Date: Mon, 14 Oct 2024 14:22:38 +0200 Subject: [PATCH 6/8] chore: Add link about install instructions Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --- java-base/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/java-base/Dockerfile b/java-base/Dockerfile index 9966e0611..c752b415b 100644 --- a/java-base/Dockerfile +++ b/java-base/Dockerfile @@ -17,6 +17,7 @@ LABEL name="Stackable image for OpenJDK" \ summary="The Stackable OpenJDK base image." \ description="This image is the base image for all Stackable Java product images." +# See: https://adoptium.net/en-gb/installation/linux/#_centosrhelfedora_instructions RUN cat < /etc/yum.repos.d/adoptium.repo [Adoptium] name=Adoptium From cc96a0b7a8ddfc04de830344ed08622983bd0bfd Mon Sep 17 00:00:00 2001 From: Nick <10092581+NickLarsenNZ@users.noreply.github.com> Date: Mon, 14 Oct 2024 14:40:20 +0200 Subject: [PATCH 7/8] Apply suggestions from code review Co-authored-by: Techassi --- java-devel/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/java-devel/Dockerfile b/java-devel/Dockerfile index 304165b60..03223068f 100644 --- a/java-devel/Dockerfile +++ b/java-devel/Dockerfile @@ -9,6 +9,7 @@ FROM stackable/image/stackable-base ARG PRODUCT +# See: https://adoptium.net/en-gb/installation/linux/#_centosrhelfedora_instructions RUN cat < /etc/yum.repos.d/adoptium.repo [Adoptium] name=Adoptium From 625d9a03f2c90be4a99b53c5b5536d92f7d04988 Mon Sep 17 00:00:00 2001 From: Nick <10092581+NickLarsenNZ@users.noreply.github.com> Date: Tue, 15 Oct 2024 10:24:16 +0200 Subject: [PATCH 8/8] Apply suggestions from code review Co-authored-by: Lars Francke --- stackable-base/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stackable-base/Dockerfile b/stackable-base/Dockerfile index 84989120a..b7184dcc3 100644 --- a/stackable-base/Dockerfile +++ b/stackable-base/Dockerfile @@ -81,10 +81,10 @@ RUN <