Add Greengage, WarehousePG, and OpenGPDB

greengage/README.md

@@ -0,0 +1,17 @@
+Greengage DB is an open-source MPP database based on Greenplum,
+maintained by the GreengageDB community after Greenplum went closed
+source in 2024. https://github.com/GreengageDB/greengage
+
+The benchmark runs Greengage inside a privileged Ubuntu 22.04 docker
+container, so the host can be any distro with docker. The install
+script fetches the upstream `greengage6.deb` (no source build) and
+initializes a single-host, 14-segment cluster via gpinitsystem.
+
+The other scripts (start/stop/check/load/query) `docker exec` into the
+running container.
+
+To run the test:
+
+```
+./benchmark.sh
+```

greengage/benchmark.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# Thin shim — actual flow is in lib/benchmark-common.sh.
+export BENCH_DOWNLOAD_SCRIPT="download-hits-tsv"
+export BENCH_DURABLE=yes
+exec ../lib/benchmark-common.sh

greengage/check

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+
+NAME=clickbench-greengage
+
+sudo docker exec "$NAME" sudo -iu gpadmin bash -lc \
+    'psql -d postgres -t -c "SELECT 1"' >/dev/null

greengage/create.sql

@@ -0,0 +1,116 @@
+drop table if exists hits;
+CREATE TABLE hits
+(
+    WatchID BIGINT NOT NULL,
+    JavaEnable SMALLINT NOT NULL,
+    Title TEXT NOT NULL,
+    GoodEvent SMALLINT NOT NULL,
+    EventTime TIMESTAMP NOT NULL,
+    EventDate Date NOT NULL,
+    CounterID INTEGER NOT NULL,
+    ClientIP INTEGER NOT NULL,
+    RegionID INTEGER NOT NULL,
+    UserID BIGINT NOT NULL,
+    CounterClass SMALLINT NOT NULL,
+    OS SMALLINT NOT NULL,
+    UserAgent SMALLINT NOT NULL,
+    URL TEXT NOT NULL,
+    Referer TEXT NOT NULL,
+    IsRefresh SMALLINT NOT NULL,
+    RefererCategoryID SMALLINT NOT NULL,
+    RefererRegionID INTEGER NOT NULL,
+    URLCategoryID SMALLINT NOT NULL,
+    URLRegionID INTEGER NOT NULL,
+    ResolutionWidth SMALLINT NOT NULL,
+    ResolutionHeight SMALLINT NOT NULL,
+    ResolutionDepth SMALLINT NOT NULL,
+    FlashMajor SMALLINT NOT NULL,
+    FlashMinor SMALLINT NOT NULL,
+    FlashMinor2 TEXT NOT NULL,
+    NetMajor SMALLINT NOT NULL,
+    NetMinor SMALLINT NOT NULL,
+    UserAgentMajor SMALLINT NOT NULL,
+    UserAgentMinor VARCHAR(255) NOT NULL,
+    CookieEnable SMALLINT NOT NULL,
+    JavascriptEnable SMALLINT NOT NULL,
+    IsMobile SMALLINT NOT NULL,
+    MobilePhone SMALLINT NOT NULL,
+    MobilePhoneModel TEXT NOT NULL,
+    Params TEXT NOT NULL,
+    IPNetworkID INTEGER NOT NULL,
+    TraficSourceID SMALLINT NOT NULL,
+    SearchEngineID SMALLINT NOT NULL,
+    SearchPhrase TEXT NOT NULL,
+    AdvEngineID SMALLINT NOT NULL,
+    IsArtifical SMALLINT NOT NULL,
+    WindowClientWidth SMALLINT NOT NULL,
+    WindowClientHeight SMALLINT NOT NULL,
+    ClientTimeZone SMALLINT NOT NULL,
+    ClientEventTime TIMESTAMP NOT NULL,
+    SilverlightVersion1 SMALLINT NOT NULL,
+    SilverlightVersion2 SMALLINT NOT NULL,
+    SilverlightVersion3 INTEGER NOT NULL,
+    SilverlightVersion4 SMALLINT NOT NULL,
+    PageCharset TEXT NOT NULL,
+    CodeVersion INTEGER NOT NULL,
+    IsLink SMALLINT NOT NULL,
+    IsDownload SMALLINT NOT NULL,
+    IsNotBounce SMALLINT NOT NULL,
+    FUniqID BIGINT NOT NULL,
+    OriginalURL TEXT NOT NULL,
+    HID INTEGER NOT NULL,
+    IsOldCounter SMALLINT NOT NULL,
+    IsEvent SMALLINT NOT NULL,
+    IsParameter SMALLINT NOT NULL,
+    DontCountHits SMALLINT NOT NULL,
+    WithHash SMALLINT NOT NULL,
+    HitColor CHAR NOT NULL,
+    LocalEventTime TIMESTAMP NOT NULL,
+    Age SMALLINT NOT NULL,
+    Sex SMALLINT NOT NULL,
+    Income SMALLINT NOT NULL,
+    Interests SMALLINT NOT NULL,
+    Robotness SMALLINT NOT NULL,
+    RemoteIP INTEGER NOT NULL,
+    WindowName INTEGER NOT NULL,
+    OpenerName INTEGER NOT NULL,
+    HistoryLength SMALLINT NOT NULL,
+    BrowserLanguage TEXT NOT NULL,
+    BrowserCountry TEXT NOT NULL,
+    SocialNetwork TEXT NOT NULL,
+    SocialAction TEXT NOT NULL,
+    HTTPError SMALLINT NOT NULL,
+    SendTiming INTEGER NOT NULL,
+    DNSTiming INTEGER NOT NULL,
+    ConnectTiming INTEGER NOT NULL,
+    ResponseStartTiming INTEGER NOT NULL,
+    ResponseEndTiming INTEGER NOT NULL,
+    FetchTiming INTEGER NOT NULL,
+    SocialSourceNetworkID SMALLINT NOT NULL,
+    SocialSourcePage TEXT NOT NULL,
+    ParamPrice BIGINT NOT NULL,
+    ParamOrderID TEXT NOT NULL,
+    ParamCurrency TEXT NOT NULL,
+    ParamCurrencyID SMALLINT NOT NULL,
+    OpenstatServiceName TEXT NOT NULL,
+    OpenstatCampaignID TEXT NOT NULL,
+    OpenstatAdID TEXT NOT NULL,
+    OpenstatSourceID TEXT NOT NULL,
+    UTMSource TEXT NOT NULL,
+    UTMMedium TEXT NOT NULL,
+    UTMCampaign TEXT NOT NULL,
+    UTMContent TEXT NOT NULL,
+    UTMTerm TEXT NOT NULL,
+    FromTag TEXT NOT NULL,
+    HasGCLID SMALLINT NOT NULL,
+    RefererHash BIGINT NOT NULL,
+    URLHash BIGINT NOT NULL,
+    CLID INTEGER NOT NULL
+)
+with (appendoptimized=true,orientation=column,compresstype=zstd)
+DISTRIBUTED BY (UserID);
+CREATE INDEX hits_idx on hits using btree (CounterID, EventDate, UserID, EventTime, WatchID); 
+drop external table if exists hits_ext;
+CREATE EXTERNAL TABLE hits_ext (like hits)
+LOCATION ('gpfdist://localhost:8080/hits.tsv')
+FORMAT 'TEXT';

greengage/data-size

@@ -0,0 +1,9 @@
+#!/bin/bash
+# The cluster's data lives inside the container under /data0, not on the
+# host. Run du inside the container so we see the actual bytes.
+set -eu
+
+NAME=clickbench-greengage
+
+sudo docker exec "$NAME" du -bcs /data0 2>/dev/null | \
+    awk '/total$/ { print $1 }'

greengage/gpinitsystem_config

@@ -0,0 +1,64 @@
+# FILE NAME: gpinitsystem_config
+
+# Configuration file needed by the gpinitsystem
+
+################################################
+#### REQUIRED PARAMETERS
+################################################
+
+#### Naming convention for utility-generated data directories.
+SEG_PREFIX=gpseg
+
+#### Base number by which primary segment port numbers 
+#### are calculated.
+PORT_BASE=6000
+
+#### File system location(s) where primary segment data directories 
+#### will be created. The number of locations in the list dictate
+#### the number of primary segments that will get created per
+#### physical host (if multiple addresses for a host are listed in 
+#### the hostfile, the number of segments will be spread evenly across
+#### the specified interface addresses).
+declare -a DATA_DIRECTORY=(/data0/primary /data0/primary /data0/primary /data0/primary /data0/primary /data0/primary /data0/primary /data0/primary /data0/primary /data0/primary /data0/primary /data0/primary /data0/primary /data0/primary)
+
+#### OS-configured hostname or IP address of the coordinator host.
+COORDINATOR_HOSTNAME=localhost
+
+#### File system location where the coordinator data directory 
+#### will be created.
+COORDINATOR_DIRECTORY=/data0/coordinator
+
+#### Port number for the coordinator instance. 
+COORDINATOR_PORT=5432
+
+#### Shell utility used to connect to remote hosts.
+TRUSTED_SHELL=ssh
+
+#### Default server-side character set encoding.
+ENCODING=UNICODE
+
+################################################
+#### OPTIONAL MIRROR PARAMETERS
+################################################
+
+#### Base number by which mirror segment port numbers 
+#### are calculated.
+#MIRROR_PORT_BASE=7000
+
+#### File system location(s) where mirror segment data directories 
+#### will be created. The number of mirror locations must equal the
+#### number of primary locations as specified in the 
+#### DATA_DIRECTORY parameter.
+#declare -a MIRROR_DATA_DIRECTORY=(/data1/mirror /data1/mirror /data1/mirror /data2/mirror /data2/mirror /data2/mirror)
+
+
+################################################
+#### OTHER OPTIONAL PARAMETERS
+################################################
+
+#### Create a database of this name after initialization.
+#DATABASE_NAME=name_of_database
+
+#### Specify the location of the host address file here instead of
+#### with the -h option of gpinitsystem.
+#MACHINE_LIST_FILE=/home/gpadmin/gpconfigs/hostfile_gpinitsystem

greengage/install

@@ -0,0 +1,129 @@
+#!/bin/bash
+# Greengage's prebuilt .deb (greengage6.deb) targets Ubuntu 22.04 (libssl3,
+# libperl5.34, etc.) and the cluster init requires kernel tuning, sshd, and
+# the gpadmin user — none of which we want to impose on the benchmark host.
+# Run everything inside a privileged ubuntu:22.04 container instead; the
+# other scripts (start/stop/check/load/query) docker-exec into it.
+#
+# Idempotent: re-running this script after a successful install does nothing.
+set -eu
+
+NAME=clickbench-greengage
+IMAGE=ubuntu:22.04
+GG_VERSION=6.30.1
+DEB_URL="https://github.com/GreengageDB/greengage/releases/download/${GG_VERSION}/greengage6.deb"
+
+# 1. Make sure docker is installed and the daemon is up.
+if ! command -v docker >/dev/null 2>&1; then
+    if command -v apt-get >/dev/null 2>&1; then
+        sudo apt-get update -y
+        sudo apt-get install -y docker.io
+    elif command -v yum >/dev/null 2>&1; then
+        sudo yum install -y docker
+    else
+        echo "greengage/install: install docker manually first" >&2
+        exit 1
+    fi
+fi
+sudo systemctl start docker 2>/dev/null || sudo service docker start || true
+
+# 2. Skip if the container is already created and the cluster is initialized.
+if sudo docker container inspect "$NAME" >/dev/null 2>&1; then
+    if sudo docker exec "$NAME" test -d /data0/coordinator/gpseg-1 2>/dev/null; then
+        echo "greengage: container '$NAME' already initialized; nothing to do" >&2
+        exit 0
+    fi
+fi
+
+sudo docker pull "$IMAGE"
+
+# 3. Long-lived privileged container. Bridge mode + --hostname localhost so
+# gpinitsystem's ssh-to-localhost lands on the container's own sshd. Publish
+# 5432 so clients on the host can also reach postgres if desired.
+if ! sudo docker container inspect "$NAME" >/dev/null 2>&1; then
+    sudo docker run -d --privileged --hostname localhost -p 5432:5432 \
+        --name "$NAME" "$IMAGE" sleep infinity
+fi
+
+# 4. Inside the container: install the .deb, kernel tuning, gpadmin user,
+# sshd. The .deb itself ships no postinst, so the gpadmin user has to be
+# created by us.
+sudo docker exec -i "$NAME" bash -s <<EOF
+set -eu
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get update -y
+apt-get install -y curl ca-certificates sudo openssh-server openssh-client \
+    iproute2 iputils-ping less net-tools rsync zip libcap2-bin procps
+
+# Greengage 6 still needs Python 2 for gpinitsystem / gpstart.
+apt-get install -y python2 python2.7 python-is-python2 || true
+
+# Fetch and install the prebuilt .deb. apt resolves the long Depends list.
+curl -fsSL -o /tmp/greengage6.deb "${DEB_URL}"
+apt-get install -y /tmp/greengage6.deb
+rm -f /tmp/greengage6.deb
+
+# Kernel tuning. --privileged means these affect the host kernel;
+# gpinitsystem checks them so they must be set.
+SHMALL=\$(expr \$(getconf _PHYS_PAGES) / 2)
+SHMAX=\$(expr \$(getconf _PHYS_PAGES) / 2 \* \$(getconf PAGE_SIZE))
+sysctl -w kernel.shmall=\$SHMALL >/dev/null
+sysctl -w kernel.shmmax=\$SHMAX >/dev/null
+sysctl -w kernel.shmmni=4096 >/dev/null
+sysctl -w vm.overcommit_memory=2 >/dev/null
+sysctl -w vm.overcommit_ratio=95 >/dev/null
+sysctl -w net.ipv4.ip_local_port_range="10000 65535" >/dev/null
+sysctl -w kernel.sem="250 2048000 200 8192" >/dev/null
+
+cat > /etc/security/limits.d/99-gpadmin.conf <<LIMITS
+* soft nofile 524288
+* hard nofile 524288
+* soft nproc 131072
+* hard nproc 131072
+LIMITS
+
+# gpadmin user with passwordless ssh to localhost (gpinitsystem requirement).
+groupadd -f gpadmin
+id -u gpadmin >/dev/null 2>&1 || useradd gpadmin -m -g gpadmin -s /bin/bash
+echo "gpadmin ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/gpadmin
+sudo -u gpadmin bash -c '
+    set -e
+    mkdir -p ~/.ssh
+    chmod 700 ~/.ssh
+    [ -f ~/.ssh/id_rsa ] || ssh-keygen -t rsa -b 4096 -N "" -f ~/.ssh/id_rsa
+    cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys
+    chmod 600 ~/.ssh/authorized_keys
+'
+
+# /bin/ping in this image has no caps, so even root can't open a raw socket.
+# gpinitsystem pings localhost to verify reachability.
+setcap cap_net_raw+ep /bin/ping
+[ -e /bin/ping6 ] && setcap cap_net_raw+ep /bin/ping6 || true
+
+# sshd setup.
+ssh-keygen -A
+mkdir -p /run/sshd
+echo "PasswordAuthentication no" >> /etc/ssh/sshd_config
+echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
+/usr/sbin/sshd
+
+# Data dirs and gpadmin shell env.
+mkdir -p /data0/primary /data0/coordinator
+chown -R gpadmin:gpadmin /data0
+grep -q greengage_path /home/gpadmin/.bashrc || \
+    echo "source /opt/greengagedb/greengage/greengage_path.sh" >> /home/gpadmin/.bashrc
+grep -q COORDINATOR_DATA_DIRECTORY /home/gpadmin/.bashrc || \
+    echo "export COORDINATOR_DATA_DIRECTORY=/data0/coordinator/gpseg-1" >> /home/gpadmin/.bashrc
+
+echo localhost > /home/gpadmin/hosts
+chown gpadmin:gpadmin /home/gpadmin/hosts
+EOF
+
+# 5. Drop gpinitsystem_config in and run gpinitsystem.
+sudo docker cp "$(dirname "$0")/gpinitsystem_config" \
+    "$NAME":/home/gpadmin/gpinitsystem_config
+sudo docker exec "$NAME" chown gpadmin:gpadmin /home/gpadmin/gpinitsystem_config
+
+sudo docker exec "$NAME" sudo -iu gpadmin bash -lc \
+    'gpinitsystem -a -c gpinitsystem_config -h hosts'

greengage/load

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -eu
+
+NAME=clickbench-greengage
+
+# Stream hits.tsv and create.sql into the container via tar so that any
+# symlinks on the host (e.g. hits.tsv -> /elsewhere) are dereferenced.
+# `tar -h` follows symlinks; piping through `docker exec -i ... tar`
+# avoids `docker cp`'s symlink-passes-through behavior.
+tar -ch hits.tsv create.sql | \
+    sudo docker exec -i "$NAME" tar -xC /home/gpadmin
+sudo docker exec "$NAME" chown gpadmin:gpadmin \
+    /home/gpadmin/hits.tsv /home/gpadmin/create.sql
+
+sudo docker exec "$NAME" sudo -iu gpadmin bash -lc \
+    'psql -d postgres -v ON_ERROR_STOP=1 -f /home/gpadmin/create.sql'
+
+# gpfdist serves hits.tsv to the gpfdist:// foreign table referenced by
+# hits_ext. Spawn it once; subsequent loads reuse the running daemon.
+sudo docker exec "$NAME" sudo -iu gpadmin bash -lc \
+    'pgrep -u gpadmin gpfdist || nohup gpfdist -d /home/gpadmin >/tmp/gpfdist.log 2>&1 &'
+
+sudo docker exec "$NAME" sudo -iu gpadmin bash -lc \
+    'psql -d postgres -v ON_ERROR_STOP=1 -t -c "INSERT INTO hits SELECT * FROM hits_ext;"'
+sudo docker exec "$NAME" sudo -iu gpadmin bash -lc \
+    'psql -d postgres -v ON_ERROR_STOP=1 -t -c "ANALYZE hits;"'
+
+sudo docker exec "$NAME" rm -f /home/gpadmin/hits.tsv
+rm -f hits.tsv
+sync