From f037bb370f447db0a8f95870206434f708875d3c Mon Sep 17 00:00:00 2001 From: Anton Kirilov Date: Sat, 10 Dec 2022 18:05:32 +0000 Subject: [PATCH 1/4] Update MongoDB to version 6.0 and PostgreSQL to version 15 --- toolset/databases/mongodb/create.js | 27 ++++++++++--------- toolset/databases/mongodb/mongodb.dockerfile | 15 ++++++----- toolset/databases/postgres/pgdg.list | 2 -- .../databases/postgres/postgres.dockerfile | 18 ++++++------- toolset/databases/postgres/postgresql.conf | 2 +- 5 files changed, 32 insertions(+), 32 deletions(-) delete mode 100644 toolset/databases/postgres/pgdg.list diff --git a/toolset/databases/mongodb/create.js b/toolset/databases/mongodb/create.js index dc1a1e6ca61..c34e2530232 100644 --- a/toolset/databases/mongodb/create.js +++ b/toolset/databases/mongodb/create.js @@ -1,7 +1,8 @@ +disableTelemetry() db = db.getSiblingDB('hello_world') db.world.drop() for (var i = 1; i <= 10000; i++) { - db.world.save( { _id: i, id: i, randomNumber: Math.min(Math.floor(Math.random() * 10000) + 1, 10000) }) + db.world.insertOne( { _id: i, id: i, randomNumber: Math.min(Math.floor(Math.random() * 10000) + 1, 10000) }) } db.world.createIndex({_id: 1}) @@ -9,17 +10,17 @@ db.world.createIndex({id: 1}) db.fortune.drop() -db.fortune.save( {_id: 1, id: 1, message: 'fortune: No such file or directory'} ); -db.fortune.save( {_id: 2, id: 2, message: "A computer scientist is someone who fixes things that aren't broken."} ); -db.fortune.save( {_id: 3, id: 3, message: 'After enough decimal places, nobody gives a damn.'} ); -db.fortune.save( {_id: 4, id: 4, message: 'A bad random number generator: 1, 1, 1, 1, 1, 4.33e+67, 1, 1, 1'} ); -db.fortune.save( {_id: 5, id: 5, message: 'A computer program does what you tell it to do, not what you want it to do.'} ); -db.fortune.save( {_id: 6, id: 6, message: 'Emacs is a nice operating system, but I prefer UNIX. — Tom Christaensen'} ); -db.fortune.save( {_id: 7, id: 7, message: 'Any program that runs right is obsolete.'} ); -db.fortune.save( {_id: 8, id: 8, message: 'A list is only as strong as its weakest link. — Donald Knuth'} ); -db.fortune.save( {_id: 9, id: 9, message: 'Feature: A bug with seniority.'} ); -db.fortune.save( {_id: 10, id: 10, message: 'Computers make very fast, very accurate mistakes.'} ); -db.fortune.save( {_id: 11, id: 11, message: ''} ); -db.fortune.save( {_id: 12, id: 12, message: 'フレームワークのベンチマーク'} ); +db.fortune.insertOne( {_id: 1, id: 1, message: 'fortune: No such file or directory'} ); +db.fortune.insertOne( {_id: 2, id: 2, message: "A computer scientist is someone who fixes things that aren't broken."} ); +db.fortune.insertOne( {_id: 3, id: 3, message: 'After enough decimal places, nobody gives a damn.'} ); +db.fortune.insertOne( {_id: 4, id: 4, message: 'A bad random number generator: 1, 1, 1, 1, 1, 4.33e+67, 1, 1, 1'} ); +db.fortune.insertOne( {_id: 5, id: 5, message: 'A computer program does what you tell it to do, not what you want it to do.'} ); +db.fortune.insertOne( {_id: 6, id: 6, message: 'Emacs is a nice operating system, but I prefer UNIX. — Tom Christaensen'} ); +db.fortune.insertOne( {_id: 7, id: 7, message: 'Any program that runs right is obsolete.'} ); +db.fortune.insertOne( {_id: 8, id: 8, message: 'A list is only as strong as its weakest link. — Donald Knuth'} ); +db.fortune.insertOne( {_id: 9, id: 9, message: 'Feature: A bug with seniority.'} ); +db.fortune.insertOne( {_id: 10, id: 10, message: 'Computers make very fast, very accurate mistakes.'} ); +db.fortune.insertOne( {_id: 11, id: 11, message: ''} ); +db.fortune.insertOne( {_id: 12, id: 12, message: 'フレームワークのベンチマーク'} ); db.fortune.createIndex({_id: 1}) diff --git a/toolset/databases/mongodb/mongodb.dockerfile b/toolset/databases/mongodb/mongodb.dockerfile index 63931966a16..0186d3361af 100644 --- a/toolset/databases/mongodb/mongodb.dockerfile +++ b/toolset/databases/mongodb/mongodb.dockerfile @@ -1,17 +1,20 @@ -FROM buildpack-deps:bionic +FROM buildpack-deps:jammy COPY ./ ./ -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 4B7C549A058F8B6B -RUN echo "deb https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.2 multiverse" | tee /etc/apt/sources.list.d/mongodb-org.list -RUN apt-get -yqq update > /dev/null +ENV MONGODB_VERSION 6.0 + +ENV DEBIAN_FRONTEND noninteractive +RUN wget -qO - https://www.mongodb.org/static/pgp/server-${MONGODB_VERSION}.asc > /etc/apt/keyrings/mongodb-org.asc +RUN echo "deb [ signed-by=/etc/apt/keyrings/mongodb-org.asc ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/$MONGODB_VERSION multiverse" > \ + /etc/apt/sources.list.d/mongodb-org.list # Complete and utter hax if works RUN ln -s /bin/echo /bin/systemctl -RUN DEBIAN_FRONTEND=noninteractive apt-get -yqq install apt-transport-https mongodb-org > /dev/null +RUN apt-get -yqq update && apt-get -yqq install mongodb-org RUN mkdir -p /data/db RUN chmod 777 /data/db -RUN mongod --fork --logpath /var/log/mongodb.log --bind_ip_all && sleep 10 && mongo < create.js && sleep 10 +RUN mongod --fork --logpath /var/log/mongodb.log --bind_ip_all && sleep 10 && mongosh < create.js && sleep 10 CMD ["mongod", "--bind_ip_all"] diff --git a/toolset/databases/postgres/pgdg.list b/toolset/databases/postgres/pgdg.list deleted file mode 100644 index 64b2188359e..00000000000 --- a/toolset/databases/postgres/pgdg.list +++ /dev/null @@ -1,2 +0,0 @@ -deb http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main -deb-src http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main diff --git a/toolset/databases/postgres/postgres.dockerfile b/toolset/databases/postgres/postgres.dockerfile index f00cf2b0f8c..9dde8ae2473 100644 --- a/toolset/databases/postgres/postgres.dockerfile +++ b/toolset/databases/postgres/postgres.dockerfile @@ -1,31 +1,29 @@ -FROM buildpack-deps:bionic +FROM buildpack-deps:jammy ADD postgresql.conf postgresql.conf ADD pg_hba.conf pg_hba.conf ADD 60-postgresql-shm.conf 60-postgresql-shm.conf ADD create-postgres-database.sql create-postgres-database.sql ADD create-postgres.sql create-postgres.sql -ADD pgdg.list pgdg.list +ENV DEBIAN_FRONTEND noninteractive # prepare PostgreSQL APT repository -RUN cp pgdg.list /etc/apt/sources.list.d/ -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - -RUN apt-get -yqq update > /dev/null -RUN apt-get -yqq install locales +RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc > /etc/apt/keyrings/postgresql.asc +RUN echo "deb [ signed-by=/etc/apt/keyrings/postgresql.asc ] http://apt.postgresql.org/pub/repos/apt jammy-pgdg main" > \ + /etc/apt/sources.list.d/pgdg.list +RUN apt-get -yqq update && apt-get -yqq install apt-utils locales -ENV PG_VERSION 14 +ENV PG_VERSION 15 RUN locale-gen en_US.UTF-8 ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 -ENV DEBIAN_FRONTEND noninteractive # install postgresql on database machine RUN apt-get -yqq install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" postgresql-${PG_VERSION} postgresql-contrib-${PG_VERSION} # Make sure all the configuration files in main belong to postgres -RUN sed -i "s|PG_VERSION|${PG_VERSION}|g" postgresql.conf +RUN sed -i "s|PG_VERSION|${PG_VERSION}|g" postgresql.conf RUN mv postgresql.conf /etc/postgresql/${PG_VERSION}/main/postgresql.conf RUN mv pg_hba.conf /etc/postgresql/${PG_VERSION}/main/pg_hba.conf diff --git a/toolset/databases/postgres/postgresql.conf b/toolset/databases/postgres/postgresql.conf index 8e54a812f2f..fd90cd2b15b 100644 --- a/toolset/databases/postgres/postgresql.conf +++ b/toolset/databases/postgres/postgresql.conf @@ -46,7 +46,7 @@ ident_file = '/etc/postgresql/PG_VERSION/main/pg_ident.conf' # ident configurati # (change requires restart) # If external_pid_file is not explicitly set, no extra PID file is written. -external_pid_file = '/var/run/postgresql/13-main.pid' # write an extra PID file +external_pid_file = '/var/run/postgresql/PG_VERSION-main.pid' # write an extra PID file # (change requires restart) From 3d6ac7193d40dfb64172e9f65677c32cb6572850 Mon Sep 17 00:00:00 2001 From: Anton Kirilov Date: Wed, 11 Jan 2023 21:59:15 +0000 Subject: [PATCH 2/4] Revert "Revert wrk to version 4.1.0 (#7852)" This reverts commit 2e2afb8d8e7d023f7fa43ecd3c51284971a65c11. --- toolset/wrk/wrk.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolset/wrk/wrk.dockerfile b/toolset/wrk/wrk.dockerfile index 783c73307c2..bfe7f778dab 100644 --- a/toolset/wrk/wrk.dockerfile +++ b/toolset/wrk/wrk.dockerfile @@ -3,7 +3,7 @@ FROM buildpack-deps:bionic RUN apt-get update && apt-get install -yqq libluajit-5.1-dev libssl-dev luajit WORKDIR /wrk -RUN curl -sL https://github.com/wg/wrk/archive/4.1.0.tar.gz | tar xz --strip-components=1 +RUN curl -sL https://github.com/wg/wrk/archive/4.2.0.tar.gz | tar xz --strip-components=1 ENV LDFLAGS="-O3 -march=native -mtune=native -flto" ENV CFLAGS="-I /usr/include/luajit-2.1 $LDFLAGS" RUN make WITH_LUAJIT=/usr WITH_OPENSSL=/usr -j "$(nproc)" From 5714a33f29f0528d1eb4a9b8c86accd97197f8ae Mon Sep 17 00:00:00 2001 From: Anton Kirilov Date: Wed, 11 Jan 2023 21:59:47 +0000 Subject: [PATCH 3/4] Revert "Revert to using Ubuntu 18.04 for the wrk image (#7832)" This reverts commit b14756a8b4f7ae9df32ed3b3f0ee7c2865e1bf4c. --- toolset/wrk/wrk.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolset/wrk/wrk.dockerfile b/toolset/wrk/wrk.dockerfile index bfe7f778dab..9111b628a50 100644 --- a/toolset/wrk/wrk.dockerfile +++ b/toolset/wrk/wrk.dockerfile @@ -1,4 +1,4 @@ -FROM buildpack-deps:bionic +FROM buildpack-deps:jammy RUN apt-get update && apt-get install -yqq libluajit-5.1-dev libssl-dev luajit From 0df8b1ab5a9ddcaa03fdcc1977a7900db069f75c Mon Sep 17 00:00:00 2001 From: Anton Kirilov Date: Sat, 21 Jan 2023 15:29:02 +0000 Subject: [PATCH 4/4] Clean up several toolset Dockerfiles --- toolset/databases/mongodb/mongodb.dockerfile | 6 ++--- .../databases/postgres/postgres.dockerfile | 13 ++++++----- toolset/wrk/wrk.dockerfile | 23 ++++++++++--------- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/toolset/databases/mongodb/mongodb.dockerfile b/toolset/databases/mongodb/mongodb.dockerfile index 0186d3361af..36cae299628 100644 --- a/toolset/databases/mongodb/mongodb.dockerfile +++ b/toolset/databases/mongodb/mongodb.dockerfile @@ -1,10 +1,10 @@ FROM buildpack-deps:jammy -COPY ./ ./ +ARG MONGODB_VERSION=6.0 -ENV MONGODB_VERSION 6.0 +COPY ./ ./ -ENV DEBIAN_FRONTEND noninteractive +ARG DEBIAN_FRONTEND=noninteractive RUN wget -qO - https://www.mongodb.org/static/pgp/server-${MONGODB_VERSION}.asc > /etc/apt/keyrings/mongodb-org.asc RUN echo "deb [ signed-by=/etc/apt/keyrings/mongodb-org.asc ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/$MONGODB_VERSION multiverse" > \ /etc/apt/sources.list.d/mongodb-org.list diff --git a/toolset/databases/postgres/postgres.dockerfile b/toolset/databases/postgres/postgres.dockerfile index 9dde8ae2473..7e02d8d972d 100644 --- a/toolset/databases/postgres/postgres.dockerfile +++ b/toolset/databases/postgres/postgres.dockerfile @@ -1,23 +1,24 @@ FROM buildpack-deps:jammy +ARG PG_VERSION=15 + ADD postgresql.conf postgresql.conf ADD pg_hba.conf pg_hba.conf ADD 60-postgresql-shm.conf 60-postgresql-shm.conf ADD create-postgres-database.sql create-postgres-database.sql ADD create-postgres.sql create-postgres.sql -ENV DEBIAN_FRONTEND noninteractive # prepare PostgreSQL APT repository +ARG DEBIAN_FRONTEND=noninteractive RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc > /etc/apt/keyrings/postgresql.asc RUN echo "deb [ signed-by=/etc/apt/keyrings/postgresql.asc ] http://apt.postgresql.org/pub/repos/apt jammy-pgdg main" > \ /etc/apt/sources.list.d/pgdg.list RUN apt-get -yqq update && apt-get -yqq install apt-utils locales -ENV PG_VERSION 15 RUN locale-gen en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 # install postgresql on database machine RUN apt-get -yqq install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" postgresql-${PG_VERSION} postgresql-contrib-${PG_VERSION} @@ -51,6 +52,6 @@ RUN service postgresql start && \ psql -a hello_world < create-postgres.sql && \ service postgresql stop -ENV PATH $PATH:/usr/lib/postgresql/$PG_VERSION/bin +ENV PATH=$PATH:/usr/lib/postgresql/$PG_VERSION/bin CMD ["postgres"] diff --git a/toolset/wrk/wrk.dockerfile b/toolset/wrk/wrk.dockerfile index 9111b628a50..036b8e7bd19 100644 --- a/toolset/wrk/wrk.dockerfile +++ b/toolset/wrk/wrk.dockerfile @@ -1,11 +1,12 @@ FROM buildpack-deps:jammy -RUN apt-get update && apt-get install -yqq libluajit-5.1-dev libssl-dev luajit +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get -yqq update && apt-get -yqq install libluajit-5.1-dev libssl-dev luajit WORKDIR /wrk RUN curl -sL https://github.com/wg/wrk/archive/4.2.0.tar.gz | tar xz --strip-components=1 -ENV LDFLAGS="-O3 -march=native -mtune=native -flto" -ENV CFLAGS="-I /usr/include/luajit-2.1 $LDFLAGS" +ARG LDFLAGS="-O3 -march=native -mtune=native -flto" +ARG CFLAGS="-I /usr/include/luajit-2.1 $LDFLAGS" RUN make WITH_LUAJIT=/usr WITH_OPENSSL=/usr -j "$(nproc)" RUN cp wrk /usr/local/bin @@ -19,11 +20,11 @@ COPY query.sh query.sh RUN chmod 777 pipeline.lua concurrency.sh pipeline.sh query.sh # Environment vars required by the wrk scripts with nonsense defaults -ENV name name -ENV server_host server_host -ENV levels levels -ENV duration duration -ENV max_concurrency max_concurrency -ENV max_threads max_threads -ENV pipeline pipeline -ENV accept accept +ENV name=name +ENV server_host=server_host +ENV levels=levels +ENV duration=duration +ENV max_concurrency=max_concurrency +ENV max_threads=max_threads +ENV pipeline=pipeline +ENV accept=accept