From 063b0140507f3f301e09f69aa0c9326c2a6834d8 Mon Sep 17 00:00:00 2001 From: Tim DuBois Date: Tue, 31 Aug 2021 13:46:54 +0200 Subject: [PATCH 1/3] Add private registry capability --- README.md | 32 ++++++++++++++++++++++++++++++++ src/config_to_template.jl | 7 ++++++- src/generate_install_packages.jl | 11 +++++++++++ src/types_config.jl | 3 +++ 4 files changed, 52 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0a60ff4..925b5a9 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,38 @@ SimpleContainerGenerator.create_dockerfile(pkgs; run(`docker build -t my_docker_username/my_image_name .`) ``` +### Example 8 + +```julia +import SimpleContainerGenerator + +mkpath("my_image_name") +cd("my_image_name") + +# Add private registries. General will always be included as a backup. +# The first method uses your ssh credentials, whereas the second requires a +# Personal Access Token. +registry_urls = ["git@github.com:MyCompany/MyPrivateRegistry.git", + "https://username:githubPAT@github.com/MyCompany/AnotherRegistry.git"] + +# Optionally, override the URL of your privately registered package. Useful if your registry +# stores the URIs of your packages as git+ssh, but you wish to use PATs in your workflow. +pkgs = [ + (name = "Foo", url = "https://username:githubPAT@github.com/MyCompany/MyPackage.jl.git",), + (name = "Bar", ), + (name = "Baz", ), +] +julia_version = v"1.4.0" + +SimpleContainerGenerator.create_dockerfile(pkgs; + julia_version = julia_version, + output_directory = pwd(), + registry_urls = registry_urls) + +# Note: you may need to `ssh-add` your key before this command will work. +run(`docker build --ssh default -t my_docker_username/my_image_name .`) +``` + ## Docker cheatsheet | Command | Description | diff --git a/src/config_to_template.jl b/src/config_to_template.jl index c79a559..ce909e7 100644 --- a/src/config_to_template.jl +++ b/src/config_to_template.jl @@ -23,6 +23,7 @@ function Template(config::Config) apt_install = _generate_apt_install_command(config) julia_url, asc_url = _get_julia_url(config) parent_image = config.parent_image + registry_urls = config.registry_urls tests_must_pass_commands = _generate_tests_must_pass_commands(config) file_list_vector = File[ @@ -149,7 +150,11 @@ function Template(config::Config) push!(step_list_vector, RunStep("chmod 555 /usr/bin/julia")) push!(step_list_vector, RunStep("chmod 555 /usr/bin/no_sysimage_julia")) # - push!(step_list_vector, RunStep("cd /tmp; JULIA_DEBUG=all SIMPLECONTAINERGENERATOR_CONTAINER_NO_TEMP_DEPOT=\"true\" /usr/bin/no_sysimage_julia /opt/simplecontainergenerator_containers/install_packages.jl")) + if any(map(r->occursin("git@github", r), registry_urls)) + push!(step_list_vector, RunStep("mkdir -p -m 600 ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts")) + end + mount_ssh = any(map(r->occursin("git@", r), registry_urls)) ? "--mount=type=ssh " : "" + push!(step_list_vector, RunStep(mount_ssh * "cd /tmp; JULIA_DEBUG=all SIMPLECONTAINERGENERATOR_CONTAINER_NO_TEMP_DEPOT=\"true\" /usr/bin/no_sysimage_julia /opt/simplecontainergenerator_containers/install_packages.jl")) # push!(step_list_vector, RunStep("cd /tmp; JULIA_DEBUG=all /opt/bin/julia /opt/simplecontainergenerator_containers/backups_of_simplecontainergenerator_1.jl")) push!(step_list_vector, RunStep("cd /tmp; JULIA_DEBUG=all /opt/bin/julia /opt/simplecontainergenerator_containers/backups_of_simplecontainergenerator_2.jl")) diff --git a/src/generate_install_packages.jl b/src/generate_install_packages.jl index 1dcf039..ad20523 100644 --- a/src/generate_install_packages.jl +++ b/src/generate_install_packages.jl @@ -17,6 +17,15 @@ function _to_packagespec_string(pkgs::AbstractVector{<:AbstractDict}) return "Pkg.Types.PackageSpec[$(join(pkg_strings, ", "))]" end +function _to_registries_string(registry_urls::Vector{String}) + registries = String[] + for registry in registry_urls + push!(registries, "Pkg.Registry.add(Pkg.RegistrySpec(url = \"" * registry * "\"))") + end + isempty(registries) || push!(registries, "Pkg.Registry.add(Pkg.RegistrySpec(url = \"https://github.com/JuliaRegistries/General.git\"))") + return registries +end + function _generate_install_packages_content(config::Config) pkgs = config.pkgs no_test = config.no_test @@ -30,8 +39,10 @@ function _generate_install_packages_content(config::Config) end end pkgs_string = _to_packagespec_string(pkgs) + registries = _to_registries_string(config.registry_urls) lines = String[ "import Pkg", + registries..., "Pkg.add($(pkgs_string))", "for name in $(pkg_names_to_test) # pkg_names_to_test", "Pkg.add(name)", diff --git a/src/types_config.jl b/src/types_config.jl index f2077ce..f571394 100644 --- a/src/types_config.jl +++ b/src/types_config.jl @@ -3,6 +3,7 @@ struct Config exclude_packages_from_sysimage::Vector{String} julia_cpu_target::String julia_version::Union{String, VersionNumber} + registry_urls::Vector{String} make_sysimage::Bool no_test::Vector{String} packagecompiler_installation_command::String @@ -28,6 +29,7 @@ function Config(pkgs::AbstractVector{<:AbstractDict{<:Symbol,<:AbstractString}} _default_julia_cpu_target(), julia_version::Union{AbstractString, VersionNumber} = _default_julia_version(), + registry_urls = String[], make_sysimage::Bool = true, no_test = @@ -51,6 +53,7 @@ function Config(pkgs::AbstractVector{<:AbstractDict{<:Symbol,<:AbstractString}} exclude_packages_from_sysimage, julia_cpu_target, julia_version, + registry_urls, make_sysimage, no_test, packagecompiler_installation_command, From 11aa62b3c3a2ab17d42cd629df2643acbe454718 Mon Sep 17 00:00:00 2001 From: Tim DuBois Date: Tue, 31 Aug 2021 13:49:46 +0200 Subject: [PATCH 2/3] Need buildkit for this.. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 925b5a9..48b6d6c 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,7 @@ SimpleContainerGenerator.create_dockerfile(pkgs; registry_urls = registry_urls) # Note: you may need to `ssh-add` your key before this command will work. -run(`docker build --ssh default -t my_docker_username/my_image_name .`) +run(`DOCKER_BUILDKIT=1 docker build --ssh default -t my_docker_username/my_image_name .`) ``` ## Docker cheatsheet From 2045bea8447917cfa6de1103a3d2f014e1b8eeee Mon Sep 17 00:00:00 2001 From: Tim DuBois Date: Tue, 7 Sep 2021 09:21:46 +0200 Subject: [PATCH 3/3] Move mount_ssh requirements behind a flag --- README.md | 3 ++- src/config_to_template.jl | 4 ++-- src/types_config.jl | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 48b6d6c..87191ce 100644 --- a/README.md +++ b/README.md @@ -214,7 +214,8 @@ julia_version = v"1.4.0" SimpleContainerGenerator.create_dockerfile(pkgs; julia_version = julia_version, output_directory = pwd(), - registry_urls = registry_urls) + registry_urls = registry_urls, + mount_ssh = true) # Only required if using the ssh method # Note: you may need to `ssh-add` your key before this command will work. run(`DOCKER_BUILDKIT=1 docker build --ssh default -t my_docker_username/my_image_name .`) diff --git a/src/config_to_template.jl b/src/config_to_template.jl index ce909e7..1bf4607 100644 --- a/src/config_to_template.jl +++ b/src/config_to_template.jl @@ -24,6 +24,7 @@ function Template(config::Config) julia_url, asc_url = _get_julia_url(config) parent_image = config.parent_image registry_urls = config.registry_urls + mount_ssh_string = config.mount_ssh ? "--mount=type=ssh " : "" tests_must_pass_commands = _generate_tests_must_pass_commands(config) file_list_vector = File[ @@ -153,8 +154,7 @@ function Template(config::Config) if any(map(r->occursin("git@github", r), registry_urls)) push!(step_list_vector, RunStep("mkdir -p -m 600 ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts")) end - mount_ssh = any(map(r->occursin("git@", r), registry_urls)) ? "--mount=type=ssh " : "" - push!(step_list_vector, RunStep(mount_ssh * "cd /tmp; JULIA_DEBUG=all SIMPLECONTAINERGENERATOR_CONTAINER_NO_TEMP_DEPOT=\"true\" /usr/bin/no_sysimage_julia /opt/simplecontainergenerator_containers/install_packages.jl")) + push!(step_list_vector, RunStep(mount_ssh_string * "cd /tmp; JULIA_DEBUG=all SIMPLECONTAINERGENERATOR_CONTAINER_NO_TEMP_DEPOT=\"true\" /usr/bin/no_sysimage_julia /opt/simplecontainergenerator_containers/install_packages.jl")) # push!(step_list_vector, RunStep("cd /tmp; JULIA_DEBUG=all /opt/bin/julia /opt/simplecontainergenerator_containers/backups_of_simplecontainergenerator_1.jl")) push!(step_list_vector, RunStep("cd /tmp; JULIA_DEBUG=all /opt/bin/julia /opt/simplecontainergenerator_containers/backups_of_simplecontainergenerator_2.jl")) diff --git a/src/types_config.jl b/src/types_config.jl index f571394..4ecd55e 100644 --- a/src/types_config.jl +++ b/src/types_config.jl @@ -4,6 +4,7 @@ struct Config julia_cpu_target::String julia_version::Union{String, VersionNumber} registry_urls::Vector{String} + mount_ssh::Bool make_sysimage::Bool no_test::Vector{String} packagecompiler_installation_command::String @@ -30,6 +31,7 @@ function Config(pkgs::AbstractVector{<:AbstractDict{<:Symbol,<:AbstractString}} julia_version::Union{AbstractString, VersionNumber} = _default_julia_version(), registry_urls = String[], + mount_ssh::Bool = false, make_sysimage::Bool = true, no_test = @@ -54,6 +56,7 @@ function Config(pkgs::AbstractVector{<:AbstractDict{<:Symbol,<:AbstractString}} julia_cpu_target, julia_version, registry_urls, + mount_ssh, make_sysimage, no_test, packagecompiler_installation_command,