ytsaurus · achulkov2 · Apr 19, 2024
diff --git a/yt/docker/ya-build/README.md b/yt/docker/ya-build/README.md
@@ -0,0 +1,5 @@
+# Building ytsaurus images with ya make
+
+This directory contains configurations for use with the `ya package` command.
+The `ya` tool is special build instrument which can be found in the root of the repository.
+See individual README files for the specifics of each configuration and `ya package` usage instructions.
diff --git a/yt/docker/ya-build/ya.make b/yt/docker/ya-build/ya.make
@@ -0,0 +1,4 @@
+RECURSE(
+    ytsaurus
+    ytsaurus-server-override
+)
diff --git a/yt/docker/ya-build/ytsaurus-server-override/Dockerfile b/yt/docker/ya-build/ytsaurus-server-override/Dockerfile
@@ -0,0 +1,10 @@
+ARG BASE_IMAGE="dev"
+FROM ytsaurus/ytsaurus:${BASE_IMAGE}
+
+USER root
+
+# Override binaries built from source.
+COPY ./ytserver-all /usr/bin/ytserver-all
+COPY ./init_queue_agent_state.py /usr/bin/init_queue_agent_state
+COPY ./init_operations_archive.py /usr/bin/init_operations_archive
+COPY ./chyt-controller /usr/bin/chyt-controller
diff --git a/yt/docker/ya-build/ytsaurus-server-override/README.md b/yt/docker/ya-build/ytsaurus-server-override/README.md
@@ -0,0 +1,12 @@
+# Building the ytsaurus core image by overriding server binaries
+
+This configuration allows overriding the server binaries in an existing `ytsaurus` core image by local ones built from source.
+
+Example: `ytsaurus/ya package package.json --docker-registry my-registry.com --docker-build-arg "BASE_IMAGE=dev-24.1"`
+
+The base image used by default is `ytsaurus:dev`.
+The purpose of the base image is to provide yt python/cli packages with support for all drivers, including the native driver. The native driver is necessary if you are running YTsaurus with its k8s-operator.
+
+The `--docker-registry` parameter only impacts the resulting image name, which will be `my-registry.com/ytsaurus:local-<commit-SHA>` in this case.
+
+The `--custom-version` parameter can be used to override the version template specified in `package.json` with your custom string.
diff --git a/yt/docker/ya-build/ytsaurus-server-override/package.json b/yt/docker/ya-build/ytsaurus-server-override/package.json
@@ -0,0 +1,81 @@
+{
+    "meta": {
+        "name": "ytsaurus",
+        "maintainer": "YT team",
+        "description": "Core YTsaurus image built with yt client libraries from existing base image",
+        "version": "local-{revision}",
+    },
+    "build": {
+        "build_server_binaries": {
+            "targets": [
+                "yt/yt/server/all",
+                "yt/chyt/controller/cmd/chyt-controller",
+            ],
+            "build_type": "release",
+            "thinlto": true,
+            "target-platforms": [
+                "default-linux-x86_64",
+            ],
+            "flags": [
+                {
+                    "name": "NO_STRIP",
+                    "value": "yes",
+                },
+                {
+                    "name": "PIC",
+                    "value": "yes",
+                },
+            ],
+        },
+    },
+    "data": [
+        {
+            "source": {
+                "type": "ARCADIA",
+                "path": "yt/opensource-packages/ytsaurus-server-override/Dockerfile",
+            },
+            "destination": {
+                "path": "/Dockerfile",
+            },
+        },
+        {
+            "source": {
+                "type": "ARCADIA",
+                "path": "yt/python/yt/environment/init_queue_agent_state.py",
+            },
+            "destination": {
+                "path": "/",
+            },
+        },
+        {
+            "source": {
+                "type": "ARCADIA",
+                "path": "yt/python/yt/environment/init_operations_archive.py",
+            },
+            "destination": {
+                "path": "/",
+            },
+        },
+        {
+            "source": {
+                "type": "BUILD_OUTPUT",
+		        "build_key": "build_server_binaries",
+                "path": "yt/yt/server/all/ytserver-all",
+            },
+            "destination": {
+                "path": "/",
+            },
+        },
+        {
+            "source": {
+                "type": "BUILD_OUTPUT",
+		        "build_key": "build_server_binaries",
+                "path": "yt/chyt/controller/cmd/chyt-controller/chyt-controller",
+            },
+            "destination": {
+                "path": "/",
+            },
+        },
+    ],
+}
+
diff --git a/yt/docker/ya-build/ytsaurus-server-override/ya.make b/yt/docker/ya-build/ytsaurus-server-override/ya.make
@@ -0,0 +1,9 @@
+UNION()
+
+FILES(
+    Dockerfile
+
+    package.json
+)
+
+END()
diff --git a/yt/docker/ya-build/ytsaurus/Dockerfile b/yt/docker/ya-build/ytsaurus/Dockerfile
@@ -0,0 +1,121 @@
+ARG SOURCE_ROOT="/ytsaurus"
+ARG BUILD_ROOT="/build"
+
+ARG PROTOC_VERSION="3.20.1"
+
+FROM ubuntu:focal as build-python-packages
+
+ARG SOURCE_ROOT
+ENV SOURCE_ROOT $SOURCE_ROOT
+
+ARG BUILD_ROOT
+ENV BUILD_ROOT $BUILD_ROOT
+
+RUN apt-get update && apt-get install -y software-properties-common
+RUN add-apt-repository ppa:deadsnakes/ppa
+
+RUN apt-get install -y --no-install-recommends \
+      curl \
+      unzip \
+      python3 \
+      python3-pip \
+      python3.11 \
+      python3.11-venv \
+    && rm -rf /var/lib/apt/lists/*
+
+ARG PROTOC_VERSION
+ENV PROTOC_VERSION $PROTOC_VERSION
+
+RUN curl -sL -o protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip \
+    && unzip protoc.zip -d /usr/local \
+    && rm protoc.zip
+
+RUN mkdir -p ${SOURCE_ROOT}
+COPY ./yt/python ${SOURCE_ROOT}/yt/python
+COPY ./yt/yt/python ${SOURCE_ROOT}/yt/yt/python
+COPY ./certs/cacert.pem ${SOURCE_ROOT}/certs/cacert.pem
+
+RUN mkdir -p ${BUILD_ROOT}
+COPY ./artifacts/libyson_lib.so ${BUILD_ROOT}/yt/yt/python/yson_shared/libyson_lib.so
+COPY ./artifacts/libdriver_lib.so ${BUILD_ROOT}/yt/yt/python/driver/native_shared/libdriver_lib.so
+COPY ./artifacts/libdriver_rpc_lib.so ${BUILD_ROOT}/yt/yt/python/driver/rpc_shared/libdriver_rpc_lib.so
+
+RUN python3.11 -m venv package-env
+RUN . package-env/bin/activate; \
+    ${SOURCE_ROOT}/yt/python/packages/build_ytsaurus_packages.sh --ytsaurus-source-path ${SOURCE_ROOT} --ytsaurus-build-path ${BUILD_ROOT} --apply-auditwheel
+
+FROM ubuntu:focal as build-ytsaurus
+
+USER root
+
+WORKDIR /tmp
+
+# Main binary.
+COPY ./ytserver-all /usr/bin/ytserver-all
+
+# If this list changes, also update yt_nightly/Dockerfile
+RUN ln -s /usr/bin/ytserver-all /usr/bin/ytserver-master && \
+    ln -s /usr/bin/ytserver-all /usr/bin/ytserver-master-cache && \
+    ln -s /usr/bin/ytserver-all /usr/bin/ytserver-clock && \
+    ln -s /usr/bin/ytserver-all /usr/bin/ytserver-discovery && \
+    ln -s /usr/bin/ytserver-all /usr/bin/ytserver-node && \
+    ln -s /usr/bin/ytserver-all /usr/bin/ytserver-job-proxy && \
+    ln -s /usr/bin/ytserver-all /usr/bin/ytserver-exec && \
+    ln -s /usr/bin/ytserver-all /usr/bin/ytserver-proxy && \
+    ln -s /usr/bin/ytserver-all /usr/bin/ytserver-http-proxy && \
+    ln -s /usr/bin/ytserver-all /usr/bin/ytserver-tools && \
+    ln -s /usr/bin/ytserver-all /usr/bin/ytserver-scheduler && \
+    ln -s /usr/bin/ytserver-all /usr/bin/ytserver-controller-agent && \
+    ln -s /usr/bin/ytserver-all /usr/bin/ytserver-query-tracker && \
+    ln -s /usr/bin/ytserver-all /usr/bin/ytserver-queue-agent && \
+    ln -s /usr/bin/ytserver-all /usr/bin/ytserver-tcp-proxy
+
+COPY ./init_queue_agent_state.py /usr/bin/init_queue_agent_state
+COPY ./init_operations_archive.py /usr/bin/init_operations_archive
+RUN ln -s /usr/bin/init_operations_archive /usr/bin/init_operation_archive
+
+COPY ./chyt-controller /usr/bin/chyt-controller
+
+# CREDITS files with licenses for all binaries.
+COPY ./credits/ytserver-all.CREDITS /usr/bin/ytserver-all.CREDITS
+COPY ./credits/chyt-controller.CREDITS  /usr/bin/chyt-controller.CREDITS
+
+RUN apt-get update && apt-get install -y software-properties-common
+RUN add-apt-repository ppa:deadsnakes/ppa
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y \
+  containerd \
+  curl \
+  less \
+  gdb \
+  lsof \
+  strace \
+  telnet \
+  tini \
+  zstd \
+  unzip \
+  dnsutils \
+  iputils-ping \
+  lsb-release \
+  openjdk-11-jdk \
+  python3.8 \
+  python3-pip \
+  libidn11-dev \
+  python3.8-distutils
+
+COPY --from=build-python-packages ./build/ytsaurus_python /tmp/ytsaurus_python
+RUN for package in client yson local native_driver; \
+  do \
+    dist_dir="/tmp/ytsaurus_python/ytsaurus_${package}_dist"; \
+    wheel_path="${dist_dir}/$(ls ${dist_dir} | grep "^ytsaurus_$package.*whl$")"; \
+    python3.8 -m pip install ${wheel_path}; \
+  done
+
+RUN ln -s /usr/lib/jvm/java-11-openjdk-amd64 /opt/jdk11
+
+# Default python to be used by python3 jobs, for compatibility with jupyter tutorial.
+RUN ln -s /usr/bin/python3.8 /usr/bin/python3 -f
+# Force lsb_release to use python it was born to use.
+RUN sed -i 's/python3/python3.8/' /usr/bin/lsb_release
+
+RUN ln -s /usr/local/bin/yt /usr/bin/yt -f
diff --git a/yt/docker/ya-build/ytsaurus/README.md b/yt/docker/ya-build/ytsaurus/README.md
@@ -0,0 +1,23 @@
+# Building the ytsaurus core image from source
+
+This configuration allows building the core ytsaurus image from source.
+The bundled python packages are also built from source.
+
+Example: `ytsaurus/ya package package.json --docker --docker-registry my-registry.com`
+
+The `--docker-registry` parameter only impacts the resulting image name, which will be `my-registry.com/ytsaurus:local-<commit-SHA>` in this case.
+
+The `--custom-version` parameter can be used to override the version template specified in `package.json` with your custom string.
+
+## A note on python libararies
+
+YTsaurus python libraries require pre-compiled .so-files, which are built using the local python headers on your host machine.
+More specifically, `python3` and `python3-config` from your host machine are used.
+If you want to use another python version for building your image, you can override these parameters in `package.json`.
+
+### Multi-arch systems
+
+Currently, [multi-arch](https://wiki.debian.org/Multiarch/Implementation) systems and `USE_LOCAL_PYTHON=yes` in `ya make` do not work together well.
+In order for the build to work, you need to provide python-specific includes (e.g. `/usr/include/<arch-triple>/python3.x`) as a separate include-dir in such a way, that no other headers will clash.
+A simple way to do this is to copy `/usr/include/<arch-triple>/python3.x` into `/my-temp-dir/<arch-triple>/python3.x` and run the `ya package` command with `-DCFLAGS="-I/my/temp-dir`.
+