From 784730071ed9d3028c5acbb64f2720590c063433 Mon Sep 17 00:00:00 2001 From: Andrey Chulkov Date: Fri, 19 Apr 2024 14:33:03 +0000 Subject: [PATCH] Add ytsaurus core image ya package configuration --- yt/docker/ya-build/README.md | 5 + yt/docker/ya-build/ya.make | 4 + .../ytsaurus-server-override/Dockerfile | 10 + .../ytsaurus-server-override/README.md | 12 ++ .../ytsaurus-server-override/package.json | 81 ++++++++ .../ya-build/ytsaurus-server-override/ya.make | 9 + yt/docker/ya-build/ytsaurus/Dockerfile | 121 ++++++++++++ yt/docker/ya-build/ytsaurus/README.md | 23 +++ yt/docker/ya-build/ytsaurus/package.json | 185 ++++++++++++++++++ yt/docker/ya-build/ytsaurus/ya.make | 9 + yt/docker/ya.make | 1 + yt/ya.make | 1 + 12 files changed, 461 insertions(+) create mode 100644 yt/docker/ya-build/README.md create mode 100644 yt/docker/ya-build/ya.make create mode 100644 yt/docker/ya-build/ytsaurus-server-override/Dockerfile create mode 100644 yt/docker/ya-build/ytsaurus-server-override/README.md create mode 100644 yt/docker/ya-build/ytsaurus-server-override/package.json create mode 100644 yt/docker/ya-build/ytsaurus-server-override/ya.make create mode 100644 yt/docker/ya-build/ytsaurus/Dockerfile create mode 100644 yt/docker/ya-build/ytsaurus/README.md create mode 100644 yt/docker/ya-build/ytsaurus/package.json create mode 100644 yt/docker/ya-build/ytsaurus/ya.make diff --git a/yt/docker/ya-build/README.md b/yt/docker/ya-build/README.md new file mode 100644 index 000000000000..9b73074d1f97 --- /dev/null +++ b/yt/docker/ya-build/README.md @@ -0,0 +1,5 @@ +# Building ytsaurus images with ya make + +This directory contains configurations for use with the `ya package` command. +The `ya` tool is special build instrument which can be found in the root of the repository. +See individual README files for the specifics of each configuration and `ya package` usage instructions. \ No newline at end of file diff --git a/yt/docker/ya-build/ya.make b/yt/docker/ya-build/ya.make new file mode 100644 index 000000000000..0a9221d071a1 --- /dev/null +++ b/yt/docker/ya-build/ya.make @@ -0,0 +1,4 @@ +RECURSE( + ytsaurus + ytsaurus-server-override +) diff --git a/yt/docker/ya-build/ytsaurus-server-override/Dockerfile b/yt/docker/ya-build/ytsaurus-server-override/Dockerfile new file mode 100644 index 000000000000..7957108dba97 --- /dev/null +++ b/yt/docker/ya-build/ytsaurus-server-override/Dockerfile @@ -0,0 +1,10 @@ +ARG BASE_IMAGE="dev" +FROM ytsaurus/ytsaurus:${BASE_IMAGE} + +USER root + +# Override binaries built from source. +COPY ./ytserver-all /usr/bin/ytserver-all +COPY ./init_queue_agent_state.py /usr/bin/init_queue_agent_state +COPY ./init_operations_archive.py /usr/bin/init_operations_archive +COPY ./chyt-controller /usr/bin/chyt-controller diff --git a/yt/docker/ya-build/ytsaurus-server-override/README.md b/yt/docker/ya-build/ytsaurus-server-override/README.md new file mode 100644 index 000000000000..a98e20bca026 --- /dev/null +++ b/yt/docker/ya-build/ytsaurus-server-override/README.md @@ -0,0 +1,12 @@ +# Building the ytsaurus core image by overriding server binaries + +This configuration allows overriding the server binaries in an existing `ytsaurus` core image by local ones built from source. + +Example: `ytsaurus/ya package package.json --docker-registry my-registry.com --docker-build-arg "BASE_IMAGE=dev-24.1"` + +The base image used by default is `ytsaurus:dev`. +The purpose of the base image is to provide yt python/cli packages with support for all drivers, including the native driver. The native driver is necessary if you are running YTsaurus with its k8s-operator. + +The `--docker-registry` parameter only impacts the resulting image name, which will be `my-registry.com/ytsaurus:local-` in this case. + +The `--custom-version` parameter can be used to override the version template specified `package.json` with your custom string. diff --git a/yt/docker/ya-build/ytsaurus-server-override/package.json b/yt/docker/ya-build/ytsaurus-server-override/package.json new file mode 100644 index 000000000000..bcbe33da39bc --- /dev/null +++ b/yt/docker/ya-build/ytsaurus-server-override/package.json @@ -0,0 +1,81 @@ +{ + "meta": { + "name": "ytsaurus", + "maintainer": "YT team", + "description": "Core YTsaurus image built with yt client libraries from existing base image", + "version": "local-{revision}", + }, + "build": { + "build_server_binaries": { + "targets": [ + "yt/yt/server/all", + "yt/chyt/controller/cmd/chyt-controller", + ], + "build_type": "release", + "thinlto": true, + "target-platforms": [ + "default-linux-x86_64", + ], + "flags": [ + { + "name": "NO_STRIP", + "value": "yes", + }, + { + "name": "PIC", + "value": "yes", + }, + ], + }, + }, + "data": [ + { + "source": { + "type": "ARCADIA", + "path": "yt/opensource-packages/ytsaurus-server-override/Dockerfile", + }, + "destination": { + "path": "/Dockerfile", + }, + }, + { + "source": { + "type": "ARCADIA", + "path": "yt/python/yt/environment/init_queue_agent_state.py", + }, + "destination": { + "path": "/", + }, + }, + { + "source": { + "type": "ARCADIA", + "path": "yt/python/yt/environment/init_operations_archive.py", + }, + "destination": { + "path": "/", + }, + }, + { + "source": { + "type": "BUILD_OUTPUT", + "build_key": "build_server_binaries", + "path": "yt/yt/server/all/ytserver-all", + }, + "destination": { + "path": "/", + }, + }, + { + "source": { + "type": "BUILD_OUTPUT", + "build_key": "build_server_binaries", + "path": "yt/chyt/controller/cmd/chyt-controller/chyt-controller", + }, + "destination": { + "path": "/", + }, + }, + ], +} + diff --git a/yt/docker/ya-build/ytsaurus-server-override/ya.make b/yt/docker/ya-build/ytsaurus-server-override/ya.make new file mode 100644 index 000000000000..ff9ae7137990 --- /dev/null +++ b/yt/docker/ya-build/ytsaurus-server-override/ya.make @@ -0,0 +1,9 @@ +UNION() + +FILES( + Dockerfile + + package.json +) + +END() diff --git a/yt/docker/ya-build/ytsaurus/Dockerfile b/yt/docker/ya-build/ytsaurus/Dockerfile new file mode 100644 index 000000000000..b196efd5fb5e --- /dev/null +++ b/yt/docker/ya-build/ytsaurus/Dockerfile @@ -0,0 +1,121 @@ +ARG SOURCE_ROOT="/ytsaurus" +ARG BUILD_ROOT="/build" + +ARG PROTOC_VERSION="3.20.1" + +FROM ubuntu:focal as build-python-packages + +ARG SOURCE_ROOT +ENV SOURCE_ROOT $SOURCE_ROOT + +ARG BUILD_ROOT +ENV BUILD_ROOT $BUILD_ROOT + +RUN apt-get update && apt-get install -y software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa + +RUN apt-get install -y --no-install-recommends \ + curl \ + unzip \ + python3 \ + python3-pip \ + python3.11 \ + python3.11-venv \ + && rm -rf /var/lib/apt/lists/* + +ARG PROTOC_VERSION +ENV PROTOC_VERSION $PROTOC_VERSION + +RUN curl -sL -o protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip \ + && unzip protoc.zip -d /usr/local \ + && rm protoc.zip + +RUN mkdir -p ${SOURCE_ROOT} +COPY ./yt/python ${SOURCE_ROOT}/yt/python +COPY ./yt/yt/python ${SOURCE_ROOT}/yt/yt/python +COPY ./certs/cacert.pem ${SOURCE_ROOT}/certs/cacert.pem + +RUN mkdir -p ${BUILD_ROOT} +COPY ./artifacts/libyson_lib.so ${BUILD_ROOT}/yt/yt/python/yson_shared/libyson_lib.so +COPY ./artifacts/libdriver_lib.so ${BUILD_ROOT}/yt/yt/python/driver/native_shared/libdriver_lib.so +COPY ./artifacts/libdriver_rpc_lib.so ${BUILD_ROOT}/yt/yt/python/driver/rpc_shared/libdriver_rpc_lib.so + +RUN python3.11 -m venv package-env +RUN . package-env/bin/activate; \ + ${SOURCE_ROOT}/yt/python/packages/build_ytsaurus_packages.sh --ytsaurus-source-path ${SOURCE_ROOT} --ytsaurus-build-path ${BUILD_ROOT} --apply-auditwheel + +FROM ubuntu:focal as build-ytsaurus + +USER root + +WORKDIR /tmp + +# Main binary. +COPY ./ytserver-all /usr/bin/ytserver-all + +# If this list changes, also update yt_nightly/Dockerfile +RUN ln -s /usr/bin/ytserver-all /usr/bin/ytserver-master && \ + ln -s /usr/bin/ytserver-all /usr/bin/ytserver-master-cache && \ + ln -s /usr/bin/ytserver-all /usr/bin/ytserver-clock && \ + ln -s /usr/bin/ytserver-all /usr/bin/ytserver-discovery && \ + ln -s /usr/bin/ytserver-all /usr/bin/ytserver-node && \ + ln -s /usr/bin/ytserver-all /usr/bin/ytserver-job-proxy && \ + ln -s /usr/bin/ytserver-all /usr/bin/ytserver-exec && \ + ln -s /usr/bin/ytserver-all /usr/bin/ytserver-proxy && \ + ln -s /usr/bin/ytserver-all /usr/bin/ytserver-http-proxy && \ + ln -s /usr/bin/ytserver-all /usr/bin/ytserver-tools && \ + ln -s /usr/bin/ytserver-all /usr/bin/ytserver-scheduler && \ + ln -s /usr/bin/ytserver-all /usr/bin/ytserver-controller-agent && \ + ln -s /usr/bin/ytserver-all /usr/bin/ytserver-query-tracker && \ + ln -s /usr/bin/ytserver-all /usr/bin/ytserver-queue-agent && \ + ln -s /usr/bin/ytserver-all /usr/bin/ytserver-tcp-proxy + +COPY ./init_queue_agent_state.py /usr/bin/init_queue_agent_state +COPY ./init_operations_archive.py /usr/bin/init_operations_archive +RUN ln -s /usr/bin/init_operations_archive /usr/bin/init_operation_archive + +COPY ./chyt-controller /usr/bin/chyt-controller + +# CREDITS files with licenses for all binaries. +COPY ./credits/ytserver-all.CREDITS /usr/bin/ytserver-all.CREDITS +COPY ./credits/chyt-controller.CREDITS /usr/bin/chyt-controller.CREDITS + +RUN apt-get update && apt-get install -y software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y \ + containerd \ + curl \ + less \ + gdb \ + lsof \ + strace \ + telnet \ + tini \ + zstd \ + unzip \ + dnsutils \ + iputils-ping \ + lsb-release \ + openjdk-11-jdk \ + python3.8 \ + python3-pip \ + libidn11-dev \ + python3.8-distutils + +COPY --from=build-python-packages ./build/ytsaurus_python /tmp/ytsaurus_python +RUN for package in client yson local native_driver; \ + do \ + dist_dir="/tmp/ytsaurus_python/ytsaurus_${package}_dist"; \ + wheel_path="${dist_dir}/$(ls ${dist_dir} | grep "^ytsaurus_$package.*whl$")"; \ + python3.8 -m pip install ${wheel_path}; \ + done + +RUN ln -s /usr/lib/jvm/java-11-openjdk-amd64 /opt/jdk11 + +# Default python to be used by python3 jobs, for compatibility with jupyter tutorial. +RUN ln -s /usr/bin/python3.8 /usr/bin/python3 -f +# Force lsb_release to use python it was born to use. +RUN sed -i 's/python3/python3.8/' /usr/bin/lsb_release + +RUN ln -s /usr/local/bin/yt /usr/bin/yt -f diff --git a/yt/docker/ya-build/ytsaurus/README.md b/yt/docker/ya-build/ytsaurus/README.md new file mode 100644 index 000000000000..93c5a35e4b48 --- /dev/null +++ b/yt/docker/ya-build/ytsaurus/README.md @@ -0,0 +1,23 @@ +# Building the ytsaurus core image from source + +This configuration allows building the core ytsaurus image from source. +The bundled python packages are also built from source. + +Example: `ytsaurus/ya package package.json --docker --docker-registry my-registry.com` + +The `--docker-registry` parameter only impacts the resulting image name, which will be `my-registry.com/ytsaurus:local-` in this case. + +The `--custom-version` parameter can be used to override the version template specified `package.json` with your custom string. + +## A note on python libararies + +YTsaurus python libraries require pre-compiled .so-files, which are built using the local python headers on your host machine. +More specifically, `python3` and `python3-config` from your host machine are used. +If you want to use another python version for building your image, you can override these parameters in `package.json`. + +### Multi-arch systems + +Currently, [multi-arch](https://wiki.debian.org/Multiarch/Implementation) systems and `USE_LOCAL_PYTHON=yes` in `ya make` do not work together well. +In order for the build to work, you need to provide python-specific includes (e.g. `/usr/include//python3.x`) as a separate include-dir in such a way, that no other headers will clash. +A simple way to do this is to copy `/usr/include//python3.x` into `/my-temp-dir//python3.x` and run the `ya package` command with `-DCFLAGS="-I/my/temp-dir`. + diff --git a/yt/docker/ya-build/ytsaurus/package.json b/yt/docker/ya-build/ytsaurus/package.json new file mode 100644 index 000000000000..e269cbcb727d --- /dev/null +++ b/yt/docker/ya-build/ytsaurus/package.json @@ -0,0 +1,185 @@ +{ + "meta": { + "name": "ytsaurus", + "maintainer": "YT team", + "description": "Core YTsaurus image built with yt client libraries from source files", + "version": "local-{revision}", + }, + "build": { + "build_python_binaries": { + "targets": [ + "yt/yt/python/yson_shared", + "yt/yt/python/driver/native_shared", + "yt/yt/python/driver/rpc_shared", + ], + "build_type": "release", + "thinlto": true, + "target-platforms": [ + "default-linux-x86_64", + ], + "flags": [ + { + "name": "USE_ARCADIA_PYTHON", + "value": "no", + }, + { + "name": "USE_LOCAL_PYTHON", + "value": "yes", + }, + { + "name": "PYTHON_BIN", + "value": "python3", + }, + { + "name": "PYTHON_CONFIG", + "value": "python3-config", + }, + { + "name": "STRIP", + "value": "yes", + }, + { + "name": "PIC", + "value": "yes", + }, + ], + }, + "build_server_binaries": { + "targets": [ + "yt/yt/server/all", + "yt/chyt/controller/cmd/chyt-controller", + ], + "build_type": "release", + "thinlto": true, + "target-platforms": [ + "default-linux-x86_64", + ], + "flags": [ + { + "name": "NO_STRIP", + "value": "yes", + }, + { + "name": "PIC", + "value": "yes", + }, + ], + }, + }, + "data": [ + { + "source": { + "type": "ARCADIA", + "path": "yt/opensource-packages/ytsaurus/Dockerfile", + }, + "destination": { + "path": "/Dockerfile", + }, + }, + { + "source": { + "type": "ARCADIA", + "path": "certs/cacert.pem", + }, + "destination": { + "path": "/certs/cacert.pem", + }, + }, + { + "source": { + "type": "ARCADIA", + "path": "yt/python", + }, + "destination": { + "path": "/yt/python", + }, + }, + { + "source": { + "type": "ARCADIA", + "path": "yt/yt/python", + }, + "destination": { + "path": "/yt/yt/python", + }, + }, + { + "source": { + "type": "ARCADIA", + "path": "yt/docker/ytsaurus/credits", + }, + "destination": { + "path": "/credits", + }, + }, + { + "source": { + "type": "ARCADIA", + "path": "yt/python/yt/environment/init_queue_agent_state.py", + }, + "destination": { + "path": "/", + }, + }, + { + "source": { + "type": "ARCADIA", + "path": "yt/python/yt/environment/init_operations_archive.py", + }, + "destination": { + "path": "/", + }, + }, + { + "source": { + "type": "BUILD_OUTPUT", + "build_key": "build_python_binaries", + "path": "yt/yt/python/yson_shared/yson_lib.so", + }, + "destination": { + "path": "/artifacts/libyson_lib.so", + }, + }, + { + "source": { + "type": "BUILD_OUTPUT", + "build_key": "build_python_binaries", + "path": "yt/yt/python/driver/native_shared/driver_lib.so", + }, + "destination": { + "path": "/artifacts/libdriver_lib.so", + }, + }, + { + "source": { + "type": "BUILD_OUTPUT", + "build_key": "build_python_binaries", + "path": "yt/yt/python/driver/rpc_shared/driver_rpc_lib.so", + }, + "destination": { + "path": "/artifacts/libdriver_rpc_lib.so", + }, + }, + { + "source": { + "type": "BUILD_OUTPUT", + "build_key": "build_server_binaries", + "path": "yt/yt/server/all/ytserver-all", + }, + "destination": { + "path": "/", + }, + }, + { + "source": { + "type": "BUILD_OUTPUT", + "build_key": "build_server_binaries", + "path": "yt/chyt/controller/cmd/chyt-controller/chyt-controller", + }, + "destination": { + "path": "/", + }, + }, + ], +} + diff --git a/yt/docker/ya-build/ytsaurus/ya.make b/yt/docker/ya-build/ytsaurus/ya.make new file mode 100644 index 000000000000..ff9ae7137990 --- /dev/null +++ b/yt/docker/ya-build/ytsaurus/ya.make @@ -0,0 +1,9 @@ +UNION() + +FILES( + Dockerfile + + package.json +) + +END() diff --git a/yt/docker/ya.make b/yt/docker/ya.make index 138826507c9c..626a8f8a466b 100644 --- a/yt/docker/ya.make +++ b/yt/docker/ya.make @@ -5,6 +5,7 @@ RECURSE( queue-agent strawberry systest + ya-build ytsaurus ytsaurus-bundle ) diff --git a/yt/ya.make b/yt/ya.make index c131a5dcfbfd..52e5ccb83a7e 100644 --- a/yt/ya.make +++ b/yt/ya.make @@ -15,6 +15,7 @@ RECURSE( styleguide systest admin + opensource-packages )