chore(core): add proxy settings for packages

.werf/defines/packages-clean.tmpl

@@ -0,0 +1,12 @@
+{{- define "alt packages clean" }}
+- apt-get clean
+- rm --recursive --force /var/lib/apt/lists/ftp.altlinux.org* /var/cache/apt/*.bin
+  {{- if $.DistroPackagesProxy }}
+- rm --recursive --force /var/lib/apt/lists/{{ $.DistroPackagesProxy }}* 
+  {{- end }}
+{{- end }}
+
+{{- define "debian packages clean" }}
+- apt-get clean
+- find /var/lib/apt/ /var/cache/apt/ -type f -delete
+{{- end }}

.werf/defines/packages-proxies.tmpl

@@ -0,0 +1,66 @@
+{{- define "alt packages proxy" }}
+# Replace altlinux repos with our proxy
+  {{- if $.DistroPackagesProxy }}
+- sed -i "s|ftp.altlinux.org/pub/distributions/archive|{{ $.DistroPackagesProxy }}/repository/archive-ALT-Linux-APT-Repository|g" /etc/apt/sources.list.d/alt.list
+  {{- end }}
+- export DEBIAN_FRONTEND=noninteractive
+- apt-get update -y
+{{- end }}
+
+{{- define "alt dist upgrade" }}
+- apt-get dist-upgrade -y
+- find /var/cache/apt/ -type f -delete
+- rm -rf /var/log/*log /var/log/apt/* /var/lib/dpkg/*-old /var/cache/debconf/*-old
+{{- end }}
+
+{{- define "debian packages proxy" }}
+# 5 years 157680000
+- |
+    echo "Acquire::Check-Valid-Until false;" >> /etc/apt/apt.conf
+    echo "Acquire::Check-Date false;" >> /etc/apt/apt.conf
+    echo "Acquire::Max-FutureTime 157680000;" >> /etc/apt/apt.conf
+# Replace debian repos with our proxy
+  {{- if $.DistroPackagesProxy }}
+- if [ -f /etc/apt/sources.list ]; then sed -i "s|http://deb.debian.org|http://{{ $.DistroPackagesProxy }}/repository|g" /etc/apt/sources.list; fi
+- if [ -f /etc/apt/sources.list.d/debian.sources ]; then sed -i "s|http://deb.debian.org|http://{{ $.DistroPackagesProxy }}/repository|g" /etc/apt/sources.list.d/debian.sources; fi
+  {{- end }}
+- export DEBIAN_FRONTEND=noninteractive
+- apt-get update
+{{- end }}
+
+{{- define "ubuntu packages proxy" }}
+  # Replace ubuntu repos with our proxy
+  {{- if $.DistroPackagesProxy }}
+- sed -i 's|http://archive.ubuntu.com|http://{{ $.DistroPackagesProxy }}/repository/archive-ubuntu|g' /etc/apt/sources.list
+- sed -i 's|http://security.ubuntu.com|http://{{ $.DistroPackagesProxy }}/repository/security-ubuntu|g' /etc/apt/sources.list
+  {{- end }}
+- export DEBIAN_FRONTEND=noninteractive
+# one year
+- apt-get -o Acquire::Check-Valid-Until=false -o Acquire::Check-Date=false -o Acquire::Max-FutureTime=31536000 update
+{{- end }}
+
+{{- define "alpine packages proxy" }}
+# Replace alpine repos with our proxy
+  {{- if $.DistroPackagesProxy }}
+- sed -i 's|https://dl-cdn.alpinelinux.org|http://{{ $.DistroPackagesProxy }}/repository|g' /etc/apk/repositories
+  {{- end }}
+- apk update
+{{- end }}
+
+{{- define "node packages proxy" }}
+  {{- if $.DistroPackagesProxy }}
+- npm config set registry http://{{ $.DistroPackagesProxy }}/repository/npmjs/
+  {{- end }}
+{{- end }}
+
+{{- define "pypi proxy" }}
+  {{- if $.DistroPackagesProxy }}
+- |
+  cat <<"EOD" > /etc/pip.conf
+  [global]
+  index = http://{{ $.DistroPackagesProxy }}/repository/pypi-proxy/pypi
+  index-url = http://{{ $.DistroPackagesProxy }}/repository/pypi-proxy/simple
+  trusted-host = {{ $.DistroPackagesProxy }}
+  EOD
+  {{- end }}
+{{- end }}

images/base-alt-p11-binaries/werf.inc.yaml

@@ -1,4 +1,15 @@
 ---
+{{- $name := print $.ImageName "-dependencies" -}}
+{{- define "$name" -}}
+packages:
+- glibc-utils
+- mount xfsprogs xfstests util-linux e2fsprogs
+libraries:
+- libffi8 libssh-devel libssh2-devel
+{{- end -}}
+
+{{ $builderDependencies := include "$name" . | fromYaml }}
+
 image: {{ $.ImageName }}
 final: false
 fromImage: BASE_ALT_P11
@@ -9,11 +20,9 @@ git:
     - relocate_binaries.sh
 shell:
   install:
+  {{- include "alt packages proxy" . | nindent 2 }}
   - |
-    apt-get update && apt-get install -y \
-    glibc-utils \
-    libffi8 libssh-devel libssh2-devel \
-    mount xfsprogs xfstests util-linux e2fsprogs
-  - |
-    apt-get clean
-    rm --recursive --force /var/lib/apt/lists/ftp.altlinux.org* /var/cache/apt/*.bin
+    apt-get install -y \
+        {{ $builderDependencies.packages | join " " }} \
+        {{ $builderDependencies.libraries | join " " }}
+  {{- include "alt packages clean" . | nindent 2 }}

images/bounder/werf.inc.yaml

@@ -21,9 +21,11 @@ git:
         - '*.c'
 shell:
   beforeInstall:
+  {{- include "debian packages proxy" . | nindent 2 }}
   - |
-    apt-get update && apt-get install --yes gcc musl-dev musl-tools
-    apt-get clean
+    apt-get install --yes \
+      gcc musl-dev musl-tools
+  {{- include "debian packages clean" . | nindent 2 }}
   install:
   - |    
     echo "Building simple app that prints hello cdi"

images/cdi-artifact/werf.inc.yaml

@@ -1,6 +1,7 @@
 ---
 {{- $version := "1.60.3" }}
 {{- $goVersion := "1.22.7" }}
+{{- $gitRepoUrl := "kubevirt/containerized-data-importer.git" }}
 
 image: {{ $.ImageName }}
 final: false
@@ -18,67 +19,74 @@ git:
       - patches
     excludePaths:
       - patches/README.md
+secrets:
+- id: SOURCE_REPO
+  value: {{ $.SOURCE_REPO_GIT }}
 shell:
   beforeInstall:
-    - apt-get update
-    - apt-get install --yes libnbd-dev
-    - apt-get clean
-    - rm --recursive --force /var/lib/apt/lists/ftp.altlinux.org* /var/cache/apt/*.bin
+  {{- include "alt packages proxy" . | nindent 2 }}
+  - |
+    apt-get install --yes libnbd-dev
+  {{- include "alt packages clean" . | nindent 2 }}
 
   install:
-    - git clone --depth 1 --branch v{{ $version }} {{ .SOURCE_REPO }}/kubevirt/containerized-data-importer.git /containerized-data-importer
-    - cd /containerized-data-importer
-
-    - echo Download Go modules.
-    - go get golang.org/x/crypto@v0.31.0
-    - go mod download
+  - |
+    mkdir -p ~/.ssh && echo "StrictHostKeyChecking accept-new" > ~/.ssh/config
 
-    - go mod tidy
-    - go mod vendor
+    git clone --depth 1 $(cat /run/secrets/SOURCE_REPO)/{{ $gitRepoUrl }} --branch v{{ $version }} /containerized-data-importer
+
+    cd /containerized-data-importer
 
-    - |
-      for p in /patches/*.patch ; do
-        echo -n "Apply ${p} ... "
-        git apply  --ignore-space-change --ignore-whitespace ${p} && echo OK || (echo FAIL ; exit 1)
-      done
+    echo Download Go modules.
+    go get golang.org/x/crypto@v0.31.0
+    go mod download
+
+    go mod tidy
+    go mod vendor
+
+  - |
+    for p in /patches/*.patch ; do
+      echo -n "Apply ${p} ... "
+      git apply  --ignore-space-change --ignore-whitespace ${p} && echo OK || (echo FAIL ; exit 1)
+    done
 
   setup:
-    - mkdir /cdi-binaries
-    - cd /containerized-data-importer
+  - mkdir /cdi-binaries
+  - cd /containerized-data-importer
 
-    - export GO111MODULE=on
-    - export GOOS=linux
-    - export CGO_ENABLED=0
-    - export GOARCH=amd64
+  - export GO111MODULE=on
+  - export GOOS=linux
+  - export CGO_ENABLED=0
+  - export GOARCH=amd64
 
-    - echo ============== Build cdi-apiserver ===========
-    - go build -ldflags="-s -w" -o /cdi-binaries/cdi-apiserver ./cmd/cdi-apiserver
+  - echo ============== Build cdi-apiserver ===========
+  - go build -ldflags="-s -w" -o /cdi-binaries/cdi-apiserver ./cmd/cdi-apiserver
 
-    - echo ============== Build cdi-cloner ===========
-    - go build -ldflags="-s -w" -o /cdi-binaries/cdi-cloner ./cmd/cdi-cloner
+  - echo ============== Build cdi-cloner ===========
+  - go build -ldflags="-s -w" -o /cdi-binaries/cdi-cloner ./cmd/cdi-cloner
 
-    - echo ============== Build cdi-controller ===========
-    - go build -ldflags="-s -w" -o /cdi-binaries/cdi-controller ./cmd/cdi-controller
+  - echo ============== Build cdi-controller ===========
+  - go build -ldflags="-s -w" -o /cdi-binaries/cdi-controller ./cmd/cdi-controller
 
-    - echo ============== Build cdi-uploadproxy ===========
-    - go build -ldflags="-s -w" -o /cdi-binaries/cdi-uploadproxy ./cmd/cdi-uploadproxy
+  - echo ============== Build cdi-uploadproxy ===========
+  - go build -ldflags="-s -w" -o /cdi-binaries/cdi-uploadproxy ./cmd/cdi-uploadproxy
 
-    - echo ============== Build cdi-importer ===========
-    - CGO_ENABLED=1 go build -ldflags="-s -w" -o /cdi-binaries/cdi-importer ./cmd/cdi-importer
+  - echo ============== Build cdi-importer ===========
+  - CGO_ENABLED=1 go build -ldflags="-s -w" -o /cdi-binaries/cdi-importer ./cmd/cdi-importer
 
-    - echo ============== Build cdi-image-size-detection  ===========
-    - go build -ldflags="-s -w" -o /cdi-binaries/cdi-image-size-detection ./tools/cdi-image-size-detection
+  - echo ============== Build cdi-image-size-detection  ===========
+  - go build -ldflags="-s -w" -o /cdi-binaries/cdi-image-size-detection ./tools/cdi-image-size-detection
 
-    - echo ============== Build cdi-source-update-poller  ===========
-    - CGO_ENABLED=1 go build -ldflags="-s -w" -o /cdi-binaries/cdi-source-update-poller ./tools/cdi-source-update-poller
+  - echo ============== Build cdi-source-update-poller  ===========
+  - CGO_ENABLED=1 go build -ldflags="-s -w" -o /cdi-binaries/cdi-source-update-poller ./tools/cdi-source-update-poller
 
-    - echo ============== Build cdi-operator  ===========
-    - go build -ldflags="-s -w" -o /cdi-binaries/cdi-operator ./cmd/cdi-operator
+  - echo ============== Build cdi-operator  ===========
+  - go build -ldflags="-s -w" -o /cdi-binaries/cdi-operator ./cmd/cdi-operator
 
-    - strip /cdi-binaries/*
-    - chmod +x /cdi-binaries/*
-    - chown -R 64535:64535 /cdi-binaries/*
-    - ls -la /cdi-binaries
+  - strip /cdi-binaries/*
+  - chmod +x /cdi-binaries/*
+  - chown -R 64535:64535 /cdi-binaries/*
+  - ls -la /cdi-binaries
 
 ---
 image: {{ $.ImageName }}-cbuilder
@@ -92,10 +100,11 @@ git:
         - '*.c'
 shell:
   install:
+  {{- include "debian packages proxy" . | nindent 2 }}  
+  - |
+    apt-get install --yes gcc musl-dev musl-tools
+  {{- include "debian packages clean" . | nindent 2 }}
   - |
-    apt-get update && apt-get install --yes gcc musl-dev musl-tools
-    apt-get clean
-
     echo "Building simple app that prints hello cdi"
     mkdir -p /bins
     musl-gcc -static -Os -o /bins/hello hello.c

images/cdi-controller/werf.inc.yaml

@@ -45,12 +45,11 @@ import:
 # Source https://github.com/kubevirt/containerized-data-importer/blob/v1.60.3/cmd/cdi-controller/BUILD.bazel
 shell:
   install:
+  {{- include "alt packages proxy" . | nindent 2 }}
   - |
-    apt-get update && apt-get install --yes \
+    apt-get install --yes \
       {{ $cdiClonerDependencies.packages | join " " }}
-  - |
-    apt-get clean
-    rm --recursive --force /var/lib/apt/lists/ftp.altlinux.org* /var/cache/apt/*.bin
+  {{- include "alt packages clean" . | nindent 2 }}
   setup:
   - /relocate_binaries.sh -i "{{ $cdiClonerDependencies.binaries | join " " }}" -o /relocate
 # tmp folder need for ready file

images/cdi-importer/werf.inc.yaml

@@ -21,8 +21,8 @@ packages:
 binaries:
   # nbd bins and libs
   - /usr/sbin/nbdkit
-  - /usr/lib/nbdkit/filters/*
-  - /usr/lib/nbdkit/plugins/*
+  - /usr/lib64/nbdkit/filters/*.so
+  - /usr/lib64/nbdkit/plugins/*.so
   # Sqlite libs
   - /usr/lib64/libsqlite3.so.0
   # CDI binaries
@@ -58,16 +58,18 @@ import:
 - image: packages/binaries/nbdkit
   add: /nbdkit
   to: /nbdkit
-  before: setup
+  before: install
 shell:
   install:
+  {{- include "alt packages proxy" . | nindent 2 }}
   - |
-    apt-get update && apt-get install --yes \
+    apt-get install --yes \
       {{ $cdiImporterDependencies.packages | join " " }} \
       {{ $cdiImporterDependencies.libraries | join " " }}
-  - apt-get clean
-  - rm --recursive --force /var/lib/apt/lists/ftp.altlinux.org* /var/cache/apt/*.bin
-  setup:
+  {{- include "alt packages clean" . | nindent 2 }}
   - |
     cp -a /nbdkit/. /
+    rm -rf /nbdkit
+  setup:
+  - |
     /relocate_binaries.sh -i "{{ $cdiImporterDependencies.binaries | join " " }}" -o /relocate

images/distroless/werf.inc.yaml

@@ -20,24 +20,27 @@ fromImage: base-alt-p11-binaries
 final: false
 shell:
   beforeInstall:
-  - apt-get update && apt-get install ca-certificates tzdata -y
-  - apt-get clean
-  - rm --recursive --force /var/lib/apt/lists/ftp.altlinux.org* /var/cache/apt/*.bin
+  {{- include "alt packages proxy" . | nindent 2 }}
+  - |
+    apt-get install ca-certificates tzdata -y
+  {{- include "alt packages clean" . | nindent 2 }}
   install:
-  - mkdir -p /relocate/etc/{pki,ssl} /relocate/usr/{bin,sbin,share,lib,lib64}
   - |
+    mkdir -p /relocate/etc/{pki,ssl} /relocate/usr/{bin,sbin,share,lib,lib64}
+
     cd /relocate
     for dir in {bin,sbin,lib,lib64};do
       ln -s usr/$dir $dir
     done
     cd /
-  - cp -pr /tmp /relocate
-  - cp -pr /etc/passwd /etc/group /etc/hostname /etc/hosts /etc/shadow /etc/protocols /etc/services /etc/nsswitch.conf /relocate/etc
-  - cp -pr /usr/share/ca-certificates /relocate/usr/share
-  - cp -pr /usr/share/zoneinfo /relocate/usr/share
-  - cp -pr /etc/pki/tls/cert.pem /relocate/etc/ssl
-  - cp -pr /etc/pki/tls/certs /relocate/etc/ssl
-  - cp -pr /etc/pki/ca-trust/ /relocate/etc/
-  - echo "deckhouse:x:64535:64535:deckhouse:/:/sbin/nologin" >> /relocate/etc/passwd
-  - echo "deckhouse:x:64535:" >> /relocate/etc/group
-  - echo "deckhouse:!::0:::::" >> /relocate/etc/shadow
+
+    cp -pr /tmp /relocate
+    cp -pr /etc/passwd /etc/group /etc/hostname /etc/hosts /etc/shadow /etc/protocols /etc/services /etc/nsswitch.conf /relocate/etc
+    cp -pr /usr/share/ca-certificates /relocate/usr/share
+    cp -pr /usr/share/zoneinfo /relocate/usr/share
+    cp -pr /etc/pki/tls/cert.pem /relocate/etc/ssl
+    cp -pr /etc/pki/tls/certs /relocate/etc/ssl
+    cp -pr /etc/pki/ca-trust/ /relocate/etc/
+    echo "deckhouse:x:64535:64535:deckhouse:/:/sbin/nologin" >> /relocate/etc/passwd
+    echo "deckhouse:x:64535:" >> /relocate/etc/group
+    echo "deckhouse:!::0:::::" >> /relocate/etc/shadow