From b6104850fa83d19555884a1443e38dababe7df7b Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Tue, 5 Dec 2023 00:19:01 +0100 Subject: [PATCH] Automatically install latest julia version (#2046) * Automatically install latest julia version * Better text * Fix * Fix * Update setup-julia.bash * Install plumbum * Better docs * Better docs * Use subprocess.check_call instead of plumbum * Do not use dash in python filename * Remove plumbum from the image * Remove jq from the image * Remove setup-julia.bash file * Fix file name * Fix docstring --- images/datascience-notebook/Dockerfile | 2 +- images/julia-notebook/Dockerfile | 2 +- .../setup-scripts/setup-julia.bash | 40 --------- .../setup-scripts/setup_julia.py | 85 +++++++++++++++++++ tagging/get_platform.py | 5 +- 5 files changed, 91 insertions(+), 43 deletions(-) delete mode 100755 images/minimal-notebook/setup-scripts/setup-julia.bash create mode 100755 images/minimal-notebook/setup-scripts/setup_julia.py diff --git a/images/datascience-notebook/Dockerfile b/images/datascience-notebook/Dockerfile index 2d49e559eb..1d5f13adf4 100644 --- a/images/datascience-notebook/Dockerfile +++ b/images/datascience-notebook/Dockerfile @@ -27,7 +27,7 @@ ENV JULIA_DEPOT_PATH=/opt/julia \ JULIA_PKGDIR=/opt/julia # Setup Julia -RUN /opt/setup-scripts/setup-julia.bash +RUN /opt/setup-scripts/setup_julia.py USER ${NB_UID} diff --git a/images/julia-notebook/Dockerfile b/images/julia-notebook/Dockerfile index 2c982a8b17..9dbfd7fcb1 100644 --- a/images/julia-notebook/Dockerfile +++ b/images/julia-notebook/Dockerfile @@ -19,7 +19,7 @@ ENV JULIA_DEPOT_PATH=/opt/julia \ JULIA_PKGDIR=/opt/julia # Setup Julia -RUN /opt/setup-scripts/setup-julia.bash +RUN /opt/setup-scripts/setup_julia.py USER ${NB_UID} diff --git a/images/minimal-notebook/setup-scripts/setup-julia.bash b/images/minimal-notebook/setup-scripts/setup-julia.bash deleted file mode 100755 index 137b225f63..0000000000 --- a/images/minimal-notebook/setup-scripts/setup-julia.bash +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -set -exuo pipefail -# Requirements: -# - Run as the root user -# - The JULIA_PKGDIR environment variable is set - -# Default julia version to install if env var is not set -# Check https://julialang.org/downloads/ -JULIA_VERSION="${JULIA_VERSION:-1.9.3}" - -# Figure out what architecture we are installing in -JULIA_ARCH=$(uname -m) -JULIA_SHORT_ARCH="${JULIA_ARCH}" -if [ "${JULIA_SHORT_ARCH}" == "x86_64" ]; then - JULIA_SHORT_ARCH="x64" -fi - -# Figure out Julia Installer URL -JULIA_INSTALLER="julia-${JULIA_VERSION}-linux-${JULIA_ARCH}.tar.gz" -JULIA_MAJOR_MINOR=$(echo "${JULIA_VERSION}" | cut -d. -f 1,2) - -# Download and install Julia -cd /tmp -mkdir "/opt/julia-${JULIA_VERSION}" -curl --progress-bar --location --output "${JULIA_INSTALLER}" \ - "https://julialang-s3.julialang.org/bin/linux/${JULIA_SHORT_ARCH}/${JULIA_MAJOR_MINOR}/${JULIA_INSTALLER}" -tar xzf "${JULIA_INSTALLER}" -C "/opt/julia-${JULIA_VERSION}" --strip-components=1 -rm "${JULIA_INSTALLER}" - -# Link Julia installed version to /usr/local/bin, so julia launches it -ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia - -# Tell Julia where conda libraries are -mkdir -p /etc/julia -echo "push!(Libdl.DL_LOAD_PATH, \"${CONDA_DIR}/lib\")" >> /etc/julia/juliarc.jl - -# Create JULIA_PKGDIR, where user libraries are installed -mkdir "${JULIA_PKGDIR}" -chown "${NB_USER}" "${JULIA_PKGDIR}" -fix-permissions "${JULIA_PKGDIR}" diff --git a/images/minimal-notebook/setup-scripts/setup_julia.py b/images/minimal-notebook/setup-scripts/setup_julia.py new file mode 100755 index 0000000000..0cdbe0cfa2 --- /dev/null +++ b/images/minimal-notebook/setup-scripts/setup_julia.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# Requirements: +# - Run as the root user +# - The JULIA_PKGDIR environment variable is set + +import os +import platform +import shutil +import subprocess +from pathlib import Path + +import requests + + +def unify_aarch64(platform: str) -> str: + """ + Renames arm64->aarch64 to support local builds on on aarch64 Macs + """ + return { + "aarch64": "aarch64", + "arm64": "aarch64", + "x86_64": "x86_64", + }[platform] + + +def get_latest_julia_url() -> tuple[str, str]: + """ + Get the last stable version of Julia + Based on: https://github.com/JuliaLang/www.julialang.org/issues/878#issuecomment-749234813 + """ + + versions = requests.get( + "https://julialang-s3.julialang.org/bin/versions.json" + ).json() + stable_versions = {k: v for k, v in versions.items() if v["stable"]} + latest_version_files = stable_versions[max(stable_versions)]["files"] + triplet = unify_aarch64(platform.machine()) + "-linux-gnu" + file_info = [vf for vf in latest_version_files if vf["triplet"] == triplet][0] + return file_info["url"], file_info["version"] + + +def download_julia(julia_url: str) -> None: + """ + Downloads and unpacks julia + The resulting julia directory is "/opt/julia-VERSION/" + """ + tmp_file = Path("/tmp/julia.tar.gz") + subprocess.check_call( + ["curl", "--progress-bar", "--location", "--output", tmp_file, julia_url] + ) + shutil.unpack_archive(tmp_file, "/opt/") + tmp_file.unlink() + + +def prepare_julia(julia_version: str) -> None: + """ + Creates /usr/local/bin/julia symlink + Make Julia aware of conda libraries + Creates a directory for Julia user libraries + """ + # Link Julia installed version to /usr/local/bin, so julia launches it + subprocess.check_call( + ["ln", "-fs", f"/opt/julia-{julia_version}/bin/julia", "/usr/local/bin/julia"] + ) + + # Tell Julia where conda libraries are + Path("/etc/julia").mkdir() + Path("/etc/julia/juliarc.jl").write_text( + f'push!(Libdl.DL_LOAD_PATH, "{os.environ["CONDA_DIR"]}/lib")\n' + ) + + # Create JULIA_PKGDIR, where user libraries are installed + JULIA_PKGDIR = Path(os.environ["JULIA_PKGDIR"]) + JULIA_PKGDIR.mkdir() + subprocess.check_call(["chown", os.environ["NB_USER"], JULIA_PKGDIR]) + subprocess.check_call(["fix-permissions", JULIA_PKGDIR]) + + +if __name__ == "__main__": + julia_url, julia_version = get_latest_julia_url() + download_julia(julia_url=julia_url) + prepare_julia(julia_version=julia_version) diff --git a/tagging/get_platform.py b/tagging/get_platform.py index 187a259262..cda791ab7d 100644 --- a/tagging/get_platform.py +++ b/tagging/get_platform.py @@ -6,9 +6,12 @@ def unify_aarch64(platform: str) -> str: + """ + Renames arm64->aarch64 to support local builds on on aarch64 Macs + """ return { "aarch64": "aarch64", - "arm64": "aarch64", # To support local building on aarch64 Macs + "arm64": "aarch64", "x86_64": "x86_64", }[platform]