From 0fbe36b7e391b78b3d908e7db0631c3198464ab6 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 10 Apr 2025 17:12:18 +0200 Subject: [PATCH 01/12] patches I --- ...28242-Updates-async-profiler-support.patch | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/hbase/stackable/patches/2.6.1/0001-HBASE-28242-Updates-async-profiler-support.patch b/hbase/stackable/patches/2.6.1/0001-HBASE-28242-Updates-async-profiler-support.patch index 2901aaf99..bf9d654ad 100644 --- a/hbase/stackable/patches/2.6.1/0001-HBASE-28242-Updates-async-profiler-support.patch +++ b/hbase/stackable/patches/2.6.1/0001-HBASE-28242-Updates-async-profiler-support.patch @@ -4,8 +4,8 @@ Date: Tue, 6 Feb 2024 16:10:54 +0100 Subject: HBASE-28242: Updates async-profiler support --- - .../hadoop/hbase/http/ProfileServlet.java | 205 +++++++++++------- - 1 file changed, 121 insertions(+), 84 deletions(-) + .../hadoop/hbase/http/ProfileServlet.java | 207 +++++++++++------- + 1 file changed, 122 insertions(+), 85 deletions(-) diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileServlet.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileServlet.java index 83087dbed3..6139114717 100644 @@ -106,21 +106,22 @@ index 83087dbed3..6139114717 100644 ALLOC("alloc"), LOCK("lock"), - PAGE_FAULTS("page-faults"), +- CONTEXT_SWITCHES("context-switches"), +- CYCLES("cycles"), +- INSTRUCTIONS("instructions"), +- CACHE_REFERENCES("cache-references"), +- CACHE_MISSES("cache-misses"), +- BRANCHES("branches"), + ITIMER("itimer"), + BRANCH_INSTRUCTIONS("branch-instructions"), -+ BRANCH_MISSES("branch-misses"), -+ BUS_CYCLES("bus-cycles"), + BRANCH_MISSES("branch-misses"), + BUS_CYCLES("bus-cycles"), + CACHE_MISSES("cache-misses"), + CACHE_REFERENCES("cache-references"), - CONTEXT_SWITCHES("context-switches"), - CYCLES("cycles"), ++ CONTEXT_SWITCHES("context-switches"), ++ CYCLES("cycles"), + DTLB_LOAD_MISSES("dTLB-load-misses"), - INSTRUCTIONS("instructions"), -- CACHE_REFERENCES("cache-references"), -- CACHE_MISSES("cache-misses"), -- BRANCHES("branches"), -- BRANCH_MISSES("branch-misses"), -- BUS_CYCLES("bus-cycles"), ++ INSTRUCTIONS("instructions"), L1_DCACHE_LOAD_MISSES("L1-dcache-load-misses"), LLC_LOAD_MISSES("LLC-load-misses"), - DTLB_LOAD_MISSES("dTLB-load-misses"), From b5f96d547f5d38dda34212d9a1eade8f2a8262b6 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 10 Apr 2025 17:12:36 +0200 Subject: [PATCH 02/12] patches I --- .../patches/2.6.1/0005-log-statement.patch | 21 +++++++++++++++++++ .../2.6.1/0006-another-log-statement.patch | 21 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 hbase/stackable/patches/2.6.1/0005-log-statement.patch create mode 100644 hbase/stackable/patches/2.6.1/0006-another-log-statement.patch diff --git a/hbase/stackable/patches/2.6.1/0005-log-statement.patch b/hbase/stackable/patches/2.6.1/0005-log-statement.patch new file mode 100644 index 000000000..15c724ed5 --- /dev/null +++ b/hbase/stackable/patches/2.6.1/0005-log-statement.patch @@ -0,0 +1,21 @@ +From e863257c8f75ef7be7052b380d88feff3b488409 Mon Sep 17 00:00:00 2001 +From: Andrew Kenworthy +Date: Thu, 10 Apr 2025 16:46:51 +0200 +Subject: log statement + +--- + .../java/org/apache/hadoop/hbase/regionserver/HRegionServer.java | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +index 27bcef2f06..68e9f8cd06 100644 +--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java ++++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +@@ -769,6 +769,7 @@ public class HRegionServer extends Thread + + UNSAFE_RS_HOSTNAME_KEY + " is used"; + throw new IOException(msg); + } else { ++ LOG.info("Hostname: {}", rpcServices.isa.getHostName()); + return rpcServices.isa.getHostName(); + } + } else { diff --git a/hbase/stackable/patches/2.6.1/0006-another-log-statement.patch b/hbase/stackable/patches/2.6.1/0006-another-log-statement.patch new file mode 100644 index 000000000..1f03450a0 --- /dev/null +++ b/hbase/stackable/patches/2.6.1/0006-another-log-statement.patch @@ -0,0 +1,21 @@ +From 36f9523968315fa359f86ced664bd6bae9613e26 Mon Sep 17 00:00:00 2001 +From: Andrew Kenworthy +Date: Thu, 10 Apr 2025 16:53:18 +0200 +Subject: another log statement + +--- + .../java/org/apache/hadoop/hbase/regionserver/HRegionServer.java | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +index 68e9f8cd06..b632b93820 100644 +--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java ++++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +@@ -1691,6 +1691,7 @@ public class HRegionServer extends Thread + this.serverName = ServerName.valueOf(hostnameFromMasterPOV, + rpcServices.getSocketAddress().getPort(), this.startcode); + String expectedHostName = rpcServices.getSocketAddress().getHostName(); ++ LOG.info("Expected hostname: {}", expectedHostName); + // if Master use-ip is enabled, RegionServer use-ip will be enabled by default even if it + // is set to disable. so we will use the ip of the RegionServer to compare with the + // hostname passed by the Master, see HBASE-27304 for details. From 303739ee1e773369f1551ee544bed8d70ff26b15 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 23 Apr 2025 14:56:13 +0200 Subject: [PATCH 03/12] wip: patch hbase to use listener endpoints --- hbase/stackable/bin/hbase-entrypoint.sh | 9 + ...rvername-with-external-host-and-port.patch | 454 ++++++++++++++++++ .../patches/2.6.1/0005-log-statement.patch | 21 - .../2.6.1/0006-another-log-statement.patch | 21 - 4 files changed, 463 insertions(+), 42 deletions(-) create mode 100644 hbase/stackable/patches/2.6.1/0005-extend-servername-with-external-host-and-port.patch delete mode 100644 hbase/stackable/patches/2.6.1/0005-log-statement.patch delete mode 100644 hbase/stackable/patches/2.6.1/0006-another-log-statement.patch diff --git a/hbase/stackable/bin/hbase-entrypoint.sh b/hbase/stackable/bin/hbase-entrypoint.sh index 3b5192016..14d96bb61 100755 --- a/hbase/stackable/bin/hbase-entrypoint.sh +++ b/hbase/stackable/bin/hbase-entrypoint.sh @@ -16,6 +16,8 @@ HBASE_ROLE_NAME="$1" HBASE_ROLE_SERVICE_NAME="$2" # 16010 for master, 16020 for regionservers etc. HBASE_ROLE_SERVICE_PORT="$3" +# ui-http or ui-https +HBASE_PORT_NAME="$4" HBASE_ROLE_SERVICE_HOST="${HOSTNAME}.${HBASE_ROLE_SERVICE_NAME}" @@ -64,6 +66,7 @@ cp /stackable/tmp/hdfs/core-site.xml /stackable/conf cp /stackable/tmp/hbase/* /stackable/conf cp /stackable/tmp/log_config/log4j* /stackable/conf +# Kerberos if [ -f /stackable/kerberos/krb5.conf ]; then KERBEROS_REALM=$(grep -oP 'default_realm = \K.*' /stackable/kerberos/krb5.conf) export KERBEROS_REALM @@ -72,6 +75,12 @@ if [ -f /stackable/kerberos/krb5.conf ]; then sed -i -e s/\$\{env\.KERBEROS_REALM\}/"${KERBEROS_REALM}"/g /stackable/conf/hdfs-site.xml fi +# Service endpoints +HBASE_SERVICE_HOST=$(cat /stackable/listener/default-address/address) +HBASE_SERVICE_PORT=$(cat /stackable/listener/default-address/ports/"${HBASE_PORT_NAME}") +sed -i -e s/\$\{HBASE_SERVICE_HOST\}/"${HBASE_SERVICE_HOST}"/g /stackable/conf/hbase-site.xml +sed -i -e s/\$\{HBASE_SERVICE_PORT\}/"${HBASE_SERVICE_PORT}"/g /stackable/conf/hbase-site.xml + rm -f "${STACKABLE_LOG_DIR}/_vector/shutdown" prepare_signal_handlers /stackable/containerdebug --output="${STACKABLE_LOG_DIR}/containerdebug-state.json" --loop & diff --git a/hbase/stackable/patches/2.6.1/0005-extend-servername-with-external-host-and-port.patch b/hbase/stackable/patches/2.6.1/0005-extend-servername-with-external-host-and-port.patch new file mode 100644 index 000000000..91eab41ca --- /dev/null +++ b/hbase/stackable/patches/2.6.1/0005-extend-servername-with-external-host-and-port.patch @@ -0,0 +1,454 @@ +From 69c540f561f740596340b73ed1a77d5b115e1bee Mon Sep 17 00:00:00 2001 +From: Andrew Kenworthy +Date: Wed, 16 Apr 2025 16:43:33 +0200 +Subject: extend servername with external host and port + +--- + .../hbase/shaded/protobuf/ProtobufUtil.java | 15 ++++++-- + .../org/apache/hadoop/hbase/ServerName.java | 35 ++++++++++++++++++- + .../src/main/protobuf/HBase.proto | 2 ++ + .../main/protobuf/RegionServerStatus.proto | 3 ++ + hbase-protocol/src/main/protobuf/HBase.proto | 2 ++ + .../src/main/protobuf/ZooKeeper.proto | 2 ++ + .../tmpl/master/RegionServerListTmpl.jamon | 2 +- + .../apache/hadoop/hbase/master/HMaster.java | 7 ++++ + .../hadoop/hbase/master/ServerManager.java | 4 ++- + .../hbase/regionserver/HRegionServer.java | 16 ++++++++- + .../resources/hbase-webapps/master/hbck.jsp | 2 +- + .../hbase-webapps/master/rsgroup.jsp | 10 +++--- + .../resources/hbase-webapps/master/table.jsp | 22 ++++++------ + .../hbase/zookeeper/MasterAddressTracker.java | 3 ++ + 14 files changed, 102 insertions(+), 23 deletions(-) + +diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java +index 0fe181b503..be121ea641 100644 +--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java ++++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java +@@ -417,6 +417,13 @@ public final class ProtobufUtil { + if (serverName.getStartcode() >= 0) { + builder.setStartCode(serverName.getStartcode()); + } ++ ++ if (serverName.getExternalHostname() != null) { ++ builder.setExternalHostName(serverName.getExternalHostname()); ++ } ++ if (serverName.getExternalPort() != null) { ++ builder.setExternalPort(serverName.getExternalPort()); ++ } + return builder.build(); + } + +@@ -428,6 +435,9 @@ public final class ProtobufUtil { + public static ServerName toServerName(final HBaseProtos.ServerName proto) { + if (proto == null) return null; + String hostName = proto.getHostName(); ++ String externalHostName = proto.getExternalHostName(); ++ Integer externalPort = proto.getExternalPort(); ++ + long startCode = -1; + int port = -1; + if (proto.hasPort()) { +@@ -436,7 +446,7 @@ public final class ProtobufUtil { + if (proto.hasStartCode()) { + startCode = proto.getStartCode(); + } +- return ServerName.valueOf(hostName, port, startCode); ++ return ServerName.valueOf(hostName, port, startCode, externalHostName, externalPort); + } + + /** +@@ -3145,7 +3155,8 @@ public final class ProtobufUtil { + ZooKeeperProtos.Master.parser().parseFrom(data, prefixLen, data.length - prefixLen); + org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.ServerName sn = + rss.getMaster(); +- return ServerName.valueOf(sn.getHostName(), sn.getPort(), sn.getStartCode()); ++ return ServerName.valueOf(sn.getHostName(), sn.getPort(), sn.getStartCode(), ++ sn.getExternalHostName(), sn.getExternalPort()); + } catch (/* InvalidProtocolBufferException */IOException e) { + // A failed parse of the znode is pretty catastrophic. Rather than loop + // retrying hoping the bad bytes will changes, and rather than change +diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/ServerName.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/ServerName.java +index 5223bac3e5..932865ca69 100644 +--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/ServerName.java ++++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ServerName.java +@@ -85,6 +85,8 @@ public class ServerName implements Comparable, Serializable { + private final String serverName; + private final long startCode; + private transient Address address; ++ private String externalHostname; ++ private Integer externalPort; + + /** + * Cached versioned bytes of this ServerName instance. +@@ -103,11 +105,21 @@ public class ServerName implements Comparable, Serializable { + this(Address.fromParts(hostname, port), startCode); + } + +- private ServerName(final Address address, final long startCode) { ++ protected ServerName(final String hostname, final int port, final long startCode, String externalHostname, Integer externalPort) { ++ this(Address.fromParts(hostname, port), startCode, externalHostname, externalPort); ++ } ++ ++ private ServerName(final Address address, final long startCode, String externalHostname, Integer externalPort) { + // Use HostAndPort to host port and hostname. Does validation and can do ipv6 + this.address = address; + this.startCode = startCode; + this.serverName = getServerName(this.address.getHostname(), this.address.getPort(), startCode); ++ this.externalHostname = externalHostname; ++ this.externalPort = externalPort; ++ } ++ ++ private ServerName(final Address address, final long startCode) { ++ this(address, startCode, null, null); + } + + private ServerName(final String hostAndPort, final long startCode) { +@@ -177,6 +189,11 @@ public class ServerName implements Comparable, Serializable { + return INTERN_POOL.intern(new ServerName(hostname, port, startCode)); + } + ++ public static ServerName valueOf(final String hostname, final int port, final long startCode, ++ String externalHostname, Integer externalPort) { ++ return new ServerName(hostname, port, startCode, externalHostname, externalPort); ++ } ++ + /** + * Retrieve an instance of ServerName. Callers should use the equals method to compare returned + * instances, though we may return a shared immutable object as an internal optimization. +@@ -253,6 +270,22 @@ public class ServerName implements Comparable, Serializable { + return this.address.getPort(); + } + ++ public String getExternalHostname() { ++ return this.externalHostname; ++ } ++ ++ public Integer getExternalPort() { ++ return this.externalPort; ++ } ++ ++ public void setAddress(Address address) { ++ this.address = address; ++ } ++ ++ public void setExternalPort(Integer externalPort) { ++ this.externalPort = externalPort; ++ } ++ + /** + * Return the start code. + * @deprecated Since 2.5.0, will be removed in 4.0.0. Use {@link #getStartCode()} instead. +diff --git a/hbase-protocol-shaded/src/main/protobuf/HBase.proto b/hbase-protocol-shaded/src/main/protobuf/HBase.proto +index e858d7ec33..2da62e0e0a 100644 +--- a/hbase-protocol-shaded/src/main/protobuf/HBase.proto ++++ b/hbase-protocol-shaded/src/main/protobuf/HBase.proto +@@ -147,6 +147,8 @@ message ServerName { + required string host_name = 1; + optional uint32 port = 2; + optional uint64 start_code = 3; ++ optional string external_host_name = 100; ++ optional uint32 external_port = 101; + } + + // Comment data structures +diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto +index 53751082f9..ecb3cf6221 100644 +--- a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto ++++ b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto +@@ -42,6 +42,9 @@ message RegionServerStartupRequest { + + /** hostname for region server, optional */ + optional string use_this_hostname_instead = 4; ++ ++ optional string external_host_name = 100; ++ optional uint32 external_port = 101; + } + + message RegionServerStartupResponse { +diff --git a/hbase-protocol/src/main/protobuf/HBase.proto b/hbase-protocol/src/main/protobuf/HBase.proto +index ecfdc8c2c7..833cee7bee 100644 +--- a/hbase-protocol/src/main/protobuf/HBase.proto ++++ b/hbase-protocol/src/main/protobuf/HBase.proto +@@ -135,6 +135,8 @@ message ServerName { + required string host_name = 1; + optional uint32 port = 2; + optional uint64 start_code = 3; ++ optional string external_host_name = 100; ++ optional uint32 external_port = 101; + } + + // Comment data structures +diff --git a/hbase-protocol/src/main/protobuf/ZooKeeper.proto b/hbase-protocol/src/main/protobuf/ZooKeeper.proto +index c0f40e970e..bddadf4d30 100644 +--- a/hbase-protocol/src/main/protobuf/ZooKeeper.proto ++++ b/hbase-protocol/src/main/protobuf/ZooKeeper.proto +@@ -56,6 +56,8 @@ message Master { + // Major RPC version so that clients can know what version the master can accept. + optional uint32 rpc_version = 2; + optional uint32 info_port = 3; ++ optional string external_host = 4; ++ optional uint32 external_port = 5; + } + + /** +diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon +index 4c9140470b..604ecd7b2a 100644 +--- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon ++++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon +@@ -500,7 +500,7 @@ if (totalCompactingCells > 0) { + + <%java> + int infoPort = master.getRegionServerInfoPort(serverName); +- String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; ++ String url = "//" + serverName.getExternalHostname() + ":" + serverName.getExternalPort() + "/rs-status"; + + + <%if infoPort > 0%> +diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +index 3fe5abac27..652c709ab5 100644 +--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java ++++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +@@ -3128,6 +3128,13 @@ public class HMaster extends HRegionServer implements MasterServices { + + public int getRegionServerInfoPort(final ServerName sn) { + int port = this.serverManager.getInfoPort(sn); ++ if (port == 0) { ++ if (sn.getExternalPort() != null) { ++ port = sn.getExternalPort(); ++ } else { ++ port = sn.getAddress().getPort(); ++ } ++ } + return port == 0 + ? conf.getInt(HConstants.REGIONSERVER_INFO_PORT, HConstants.DEFAULT_REGIONSERVER_INFOPORT) + : port; +diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +index f7115a5cef..e5f6062793 100644 +--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java ++++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +@@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.ServerName; + import org.apache.hadoop.hbase.YouAreDeadException; + import org.apache.hadoop.hbase.client.ClusterConnection; + import org.apache.hadoop.hbase.client.RegionInfo; ++import org.apache.hadoop.hbase.client.RetriesExhaustedException; + import org.apache.hadoop.hbase.conf.ConfigurationObserver; + import org.apache.hadoop.hbase.ipc.DecommissionedHostRejectedException; + import org.apache.hadoop.hbase.ipc.HBaseRpcController; +@@ -230,7 +231,8 @@ public class ServerManager implements ConfigurationObserver { + String isaHostName = useIp ? ia.getHostAddress() : ia.getHostName(); + final String hostname = + request.hasUseThisHostnameInstead() ? request.getUseThisHostnameInstead() : isaHostName; +- ServerName sn = ServerName.valueOf(hostname, request.getPort(), request.getServerStartCode()); ++ ServerName sn = ServerName.valueOf(hostname, request.getPort(), request.getServerStartCode(), ++ request.getExternalHostName(), request.getExternalPort()); + + // Check if the host should be rejected based on it's decommissioned status + checkRejectableDecommissionedStatus(sn); +diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +index 27bcef2f06..8438378059 100644 +--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java ++++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +@@ -678,7 +678,18 @@ public class HRegionServer extends Thread + useIp ? rpcServices.isa.getAddress().getHostAddress() : rpcServices.isa.getHostName(); + String hostName = + StringUtils.isBlank(useThisHostnameInstead) ? isaHostName : useThisHostnameInstead; +- serverName = ServerName.valueOf(hostName, this.rpcServices.isa.getPort(), this.startcode); ++ ++ String externalHostname; ++ Integer externalPort; ++ if (this instanceof HMaster) { ++ externalHostname = conf.get("hbase.listener.master.hostname", ""); ++ externalPort = conf.getInt("hbase.listener.master.port", -1); ++ } else { ++ externalHostname = conf.get("hbase.listener.regionserver.hostname", ""); ++ externalPort = conf.getInt("hbase.listener.regionserver.port", -1); ++ } ++ ++ serverName = ServerName.valueOf(hostName, this.rpcServices.isa.getPort(), this.startcode, externalHostname, externalPort); + + rpcControllerFactory = RpcControllerFactory.instantiate(this.conf); + rpcRetryingCallerFactory = RpcRetryingCallerFactory.instantiate(this.conf, +@@ -3081,6 +3092,9 @@ public class HRegionServer extends Thread + request.setPort(port); + request.setServerStartCode(this.startcode); + request.setServerCurrentTime(now); ++ request.setExternalHostName(serverName.getExternalHostname()); ++ request.setExternalPort(serverName.getExternalPort()); ++ + result = rss.regionServerStartup(null, request.build()); + } catch (ServiceException se) { + IOException ioe = ProtobufUtil.getRemoteException(se); +diff --git a/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp b/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp +index 38e16ca8e2..3bd14bf524 100644 +--- a/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp ++++ b/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp +@@ -366,7 +366,7 @@ private static String formatServerName(HMaster master, + if (serverManager.isServerOnline(serverName)) { + int infoPort = master.getRegionServerInfoPort(serverName); + if (infoPort > 0) { +- return "" + sn + ""; + } else { + return "" + sn + ""; +diff --git a/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp b/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp +index 6ba6f78a27..5dad8e07a2 100644 +--- a/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp ++++ b/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp +@@ -171,7 +171,7 @@ + } + long startcode = serverName.getStartcode(); + int infoPort = master.getRegionServerInfoPort(serverName); +- String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status";%> ++ String url = "//" + serverName.getExternalHostname() + ":" + serverName.getExternalPort() + "/rs-status";%> + + <%= serverName.getServerName() %> + <%= new Date(startcode) %> +@@ -226,7 +226,7 @@ + .stream().mapToDouble(rm -> rm.getMemStoreSize().get(Size.Unit.MEGABYTE)) + .sum(); + int infoPort = master.getRegionServerInfoPort(serverName); +- String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; ++ String url = "//" + serverName.getExternalHostname() + ":" + serverName.getExternalPort() + "/rs-status"; + + if (memStoreSizeMB > 0) { + memStoreSizeMBStr = TraditionalBinaryPrefix.long2String( +@@ -277,7 +277,7 @@ + readRequestCount += rm.getReadRequestCount(); + writeRequestCount += rm.getWriteRequestCount(); + } +- String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; ++ String url = "//" + serverName.getExternalHostname() + ":" + serverName.getExternalPort() + "/rs-status"; + %> + + <%= serverName.getServerName() %> +@@ -331,7 +331,7 @@ + totalStaticBloomSizeKB += rm.getBloomFilterSize().get(Size.Unit.KILOBYTE); + } + int infoPort = master.getRegionServerInfoPort(serverName); +- String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; ++ String url = "//" + serverName.getExternalHostname() + ":" + serverName.getExternalPort() + "/rs-status"; + if (storeUncompressedSizeMB > 0) { + storeUncompressedSizeMBStr = TraditionalBinaryPrefix.long2String( + (long) storeUncompressedSizeMB * TraditionalBinaryPrefix.MEGA.value, "B", 1); +@@ -397,7 +397,7 @@ + ((float) currentCompactedCells / totalCompactingCells)) + "%"; + } + int infoPort = master.getRegionServerInfoPort(serverName); +- String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; ++ String url = "//" + serverName.getExternalHostname() + ":" + serverName.getExternalPort() + "/rs-status"; + %> + + <%= serverName.getServerName() %> +diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +index 56a0c8096d..2da74e9709 100644 +--- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp ++++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +@@ -102,7 +102,7 @@ + return serverName.getServerName(); + } + +- final String socketAddress = serverName.getHostname() + ":" + rsInfoPort; ++ final String socketAddress = serverName.getExternalHostname() + ":" + serverName.getExternalPort(); + final String URI = "//" + socketAddress + "/region.jsp" + + "?name=" + regionInfo.getEncodedName(); + return "" + serverName.getServerName() + ""; +@@ -299,7 +299,7 @@ + if (metaLocation != null) { + ServerMetrics sl = master.getServerManager().getLoad(metaLocation); + // The host name portion should be safe, but I don't know how we handle IDNs so err on the side of failing safely. +- hostAndPort = URLEncoder.encode(metaLocation.getHostname()) + ":" + master.getRegionServerInfoPort(metaLocation); ++ hostAndPort = URLEncoder.encode(metaLocation.getExternalHostname()) + ":" + master.getRegionServerInfoPort(metaLocation); + if (sl != null) { + Map map = sl.getRegionMetrics(); + if (map.containsKey(meta.getRegionName())) { +@@ -374,7 +374,7 @@ + float localityForSsd = 0.0f; + if (metaLocation != null) { + ServerMetrics sl = master.getServerManager().getLoad(metaLocation); +- hostAndPort = URLEncoder.encode(metaLocation.getHostname()) + ":" + master.getRegionServerInfoPort(metaLocation); ++ hostAndPort = URLEncoder.encode(metaLocation.getExternalHostname()) + ":" + master.getRegionServerInfoPort(metaLocation); + if (sl != null) { + Map map = sl.getRegionMetrics(); + if (map.containsKey(meta.getRegionName())) { +@@ -426,7 +426,7 @@ + String compactionProgress = ""; + if (metaLocation != null) { + ServerMetrics sl = master.getServerManager().getLoad(metaLocation); +- hostAndPort = URLEncoder.encode(metaLocation.getHostname()) + ":" + master.getRegionServerInfoPort(metaLocation); ++ hostAndPort = URLEncoder.encode(metaLocation.getExternalHostname()) + ":" + master.getRegionServerInfoPort(metaLocation); + if (sl != null) { + Map map = sl.getRegionMetrics(); + if (map.containsKey(meta.getRegionName())) { +@@ -951,7 +951,7 @@ + ServerMetrics sl = master.getServerManager().getLoad(addr); + // This port might be wrong if RS actually ended up using something else. + urlRegionServer = +- "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; ++ "//" + URLEncoder.encode(addr.getExternalHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; + if(sl != null) { + Integer i = regDistribution.get(addr); + if (null == i) i = Integer.valueOf(0); +@@ -972,7 +972,7 @@ + if (urlRegionServer != null) { + %> + +- <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> ++ <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getExternalHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> + + <% + } else { +@@ -1037,7 +1037,7 @@ + if (addr != null) { + // This port might be wrong if RS actually ended up using something else. + urlRegionServer = +- "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; ++ "//" + URLEncoder.encode(addr.getExternalHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; + } + if (numRegionsRendered < numRegionsToRender) { + numRegionsRendered++; +@@ -1098,7 +1098,7 @@ + if (addr != null) { + // This port might be wrong if RS actually ended up using something else. + urlRegionServer = +- "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; ++ "//" + URLEncoder.encode(addr.getExternalHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; + } + if (numRegionsRendered < numRegionsToRender) { + numRegionsRendered++; +@@ -1109,7 +1109,7 @@ + if (urlRegionServer != null) { + %> + +- <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> ++ <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getExternalHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> + + <% + } else { +@@ -1153,10 +1153,10 @@ + <% + for (Map.Entry rdEntry : regDistribution.entrySet()) { + ServerName addr = rdEntry.getKey(); +- String url = "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; ++ String url = "//" + URLEncoder.encode(addr.getExternalHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; + %> + +- <%= StringEscapeUtils.escapeHtml4(addr.getHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> ++ <%= StringEscapeUtils.escapeHtml4(addr.getExternalHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> + <%= rdEntry.getValue()%> + <% + if (withReplica) { +diff --git a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterAddressTracker.java b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterAddressTracker.java +index 840ee2d215..b0f9da3e00 100644 +--- a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterAddressTracker.java ++++ b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterAddressTracker.java +@@ -281,6 +281,9 @@ public class MasterAddressTracker extends ZKNodeTracker { + snbuilder.setHostName(sn.getHostname()); + snbuilder.setPort(sn.getPort()); + snbuilder.setStartCode(sn.getStartcode()); ++ snbuilder.setExternalHostName(sn.getExternalHostname()); ++ snbuilder.setExternalPort(sn.getExternalPort()); ++ + mbuilder.setMaster(snbuilder.build()); + mbuilder.setRpcVersion(HConstants.RPC_CURRENT_VERSION); + mbuilder.setInfoPort(infoPort); diff --git a/hbase/stackable/patches/2.6.1/0005-log-statement.patch b/hbase/stackable/patches/2.6.1/0005-log-statement.patch deleted file mode 100644 index 15c724ed5..000000000 --- a/hbase/stackable/patches/2.6.1/0005-log-statement.patch +++ /dev/null @@ -1,21 +0,0 @@ -From e863257c8f75ef7be7052b380d88feff3b488409 Mon Sep 17 00:00:00 2001 -From: Andrew Kenworthy -Date: Thu, 10 Apr 2025 16:46:51 +0200 -Subject: log statement - ---- - .../java/org/apache/hadoop/hbase/regionserver/HRegionServer.java | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java -index 27bcef2f06..68e9f8cd06 100644 ---- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java -+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java -@@ -769,6 +769,7 @@ public class HRegionServer extends Thread - + UNSAFE_RS_HOSTNAME_KEY + " is used"; - throw new IOException(msg); - } else { -+ LOG.info("Hostname: {}", rpcServices.isa.getHostName()); - return rpcServices.isa.getHostName(); - } - } else { diff --git a/hbase/stackable/patches/2.6.1/0006-another-log-statement.patch b/hbase/stackable/patches/2.6.1/0006-another-log-statement.patch deleted file mode 100644 index 1f03450a0..000000000 --- a/hbase/stackable/patches/2.6.1/0006-another-log-statement.patch +++ /dev/null @@ -1,21 +0,0 @@ -From 36f9523968315fa359f86ced664bd6bae9613e26 Mon Sep 17 00:00:00 2001 -From: Andrew Kenworthy -Date: Thu, 10 Apr 2025 16:53:18 +0200 -Subject: another log statement - ---- - .../java/org/apache/hadoop/hbase/regionserver/HRegionServer.java | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java -index 68e9f8cd06..b632b93820 100644 ---- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java -+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java -@@ -1691,6 +1691,7 @@ public class HRegionServer extends Thread - this.serverName = ServerName.valueOf(hostnameFromMasterPOV, - rpcServices.getSocketAddress().getPort(), this.startcode); - String expectedHostName = rpcServices.getSocketAddress().getHostName(); -+ LOG.info("Expected hostname: {}", expectedHostName); - // if Master use-ip is enabled, RegionServer use-ip will be enabled by default even if it - // is set to disable. so we will use the ip of the RegionServer to compare with the - // hostname passed by the Master, see HBASE-27304 for details. From 8dc1c99b02a83211b6963083b96a1e1de0126e37 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 23 Apr 2025 15:41:35 +0200 Subject: [PATCH 04/12] changed rendering --- ...rvername-with-external-host-and-port.patch | 101 +++++++++++++----- 1 file changed, 72 insertions(+), 29 deletions(-) diff --git a/hbase/stackable/patches/2.6.1/0005-extend-servername-with-external-host-and-port.patch b/hbase/stackable/patches/2.6.1/0005-extend-servername-with-external-host-and-port.patch index 91eab41ca..763a7c5ac 100644 --- a/hbase/stackable/patches/2.6.1/0005-extend-servername-with-external-host-and-port.patch +++ b/hbase/stackable/patches/2.6.1/0005-extend-servername-with-external-host-and-port.patch @@ -1,4 +1,4 @@ -From 69c540f561f740596340b73ed1a77d5b115e1bee Mon Sep 17 00:00:00 2001 +From 6f9e3cae1bd74e7feeca428d34f078c5542dfc2c Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 16 Apr 2025 16:43:33 +0200 Subject: extend servername with external host and port @@ -14,11 +14,11 @@ Subject: extend servername with external host and port .../apache/hadoop/hbase/master/HMaster.java | 7 ++++ .../hadoop/hbase/master/ServerManager.java | 4 ++- .../hbase/regionserver/HRegionServer.java | 16 ++++++++- - .../resources/hbase-webapps/master/hbck.jsp | 2 +- - .../hbase-webapps/master/rsgroup.jsp | 10 +++--- - .../resources/hbase-webapps/master/table.jsp | 22 ++++++------ + .../resources/hbase-webapps/master/hbck.jsp | 4 +-- + .../hbase-webapps/master/rsgroup.jsp | 15 +++----- + .../resources/hbase-webapps/master/table.jsp | 30 ++++++++-------- .../hbase/zookeeper/MasterAddressTracker.java | 3 ++ - 14 files changed, 102 insertions(+), 23 deletions(-) + 14 files changed, 107 insertions(+), 33 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java index 0fe181b503..be121ea641 100644 @@ -279,12 +279,15 @@ index 27bcef2f06..8438378059 100644 } catch (ServiceException se) { IOException ioe = ProtobufUtil.getRemoteException(se); diff --git a/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp b/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp -index 38e16ca8e2..3bd14bf524 100644 +index 38e16ca8e2..c30fd7753e 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp -@@ -366,7 +366,7 @@ private static String formatServerName(HMaster master, +@@ -364,9 +364,9 @@ private static String formatServerName(HMaster master, + } + String sn = serverName.toString(); if (serverManager.isServerOnline(serverName)) { - int infoPort = master.getRegionServerInfoPort(serverName); +- int infoPort = master.getRegionServerInfoPort(serverName); ++ int infoPort = serverName.getExternalPort(); if (infoPort > 0) { - return "" + sn + ""; diff --git a/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp b/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp -index 6ba6f78a27..5dad8e07a2 100644 +index 6ba6f78a27..f3275b9165 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp -@@ -171,7 +171,7 @@ +@@ -170,8 +170,7 @@ + lastContact = (System.currentTimeMillis() - sl.getReportTimestamp())/1000; } long startcode = serverName.getStartcode(); - int infoPort = master.getRegionServerInfoPort(serverName); +- int infoPort = master.getRegionServerInfoPort(serverName); - String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status";%> + String url = "//" + serverName.getExternalHostname() + ":" + serverName.getExternalPort() + "/rs-status";%> <%= serverName.getServerName() %> <%= new Date(startcode) %> -@@ -226,7 +226,7 @@ +@@ -225,8 +224,7 @@ + double memStoreSizeMB = sl.getRegionMetrics().values() .stream().mapToDouble(rm -> rm.getMemStoreSize().get(Size.Unit.MEGABYTE)) .sum(); - int infoPort = master.getRegionServerInfoPort(serverName); +- int infoPort = master.getRegionServerInfoPort(serverName); - String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; + String url = "//" + serverName.getExternalHostname() + ":" + serverName.getExternalPort() + "/rs-status"; if (memStoreSizeMB > 0) { memStoreSizeMBStr = TraditionalBinaryPrefix.long2String( -@@ -277,7 +277,7 @@ +@@ -270,14 +268,13 @@ + ServerName serverName = serverMaping.get(server); + ServerMetrics sl = onlineServers.get(server); + if (sl != null && serverName != null) { +- int infoPort = master.getRegionServerInfoPort(serverName); + long readRequestCount = 0; + long writeRequestCount = 0; + for (RegionMetrics rm : sl.getRegionMetrics().values()) { readRequestCount += rm.getReadRequestCount(); writeRequestCount += rm.getWriteRequestCount(); } @@ -322,28 +334,41 @@ index 6ba6f78a27..5dad8e07a2 100644 %> <%= serverName.getServerName() %> -@@ -331,7 +331,7 @@ +@@ -330,8 +327,7 @@ + totalStaticIndexSizeKB += rm.getStoreFileUncompressedDataIndexSize().get(Size.Unit.KILOBYTE); totalStaticBloomSizeKB += rm.getBloomFilterSize().get(Size.Unit.KILOBYTE); } - int infoPort = master.getRegionServerInfoPort(serverName); +- int infoPort = master.getRegionServerInfoPort(serverName); - String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; + String url = "//" + serverName.getExternalHostname() + ":" + serverName.getExternalPort() + "/rs-status"; if (storeUncompressedSizeMB > 0) { storeUncompressedSizeMBStr = TraditionalBinaryPrefix.long2String( (long) storeUncompressedSizeMB * TraditionalBinaryPrefix.MEGA.value, "B", 1); -@@ -397,7 +397,7 @@ +@@ -396,8 +392,7 @@ + percentDone = String.format("%.2f", 100 * ((float) currentCompactedCells / totalCompactingCells)) + "%"; } - int infoPort = master.getRegionServerInfoPort(serverName); +- int infoPort = master.getRegionServerInfoPort(serverName); - String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; + String url = "//" + serverName.getExternalHostname() + ":" + serverName.getExternalPort() + "/rs-status"; %> <%= serverName.getServerName() %> diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp -index 56a0c8096d..2da74e9709 100644 +index 56a0c8096d..49c8a69c60 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +@@ -93,8 +93,8 @@ + * its region server. + * @return an anchor tag if one can be built, {@code null} otherwise. + */ +- private static String buildRegionServerLink(final ServerName serverName, final int rsInfoPort, +- final RegionInfo regionInfo, final RegionState.State regionState) { ++ private static String buildRegionServerLink(final ServerName serverName, ++ final RegionInfo regionInfo, final RegionState.State regionState) { + if (serverName == null || regionInfo == null) { return null; } + + if (regionState != RegionState.State.OPEN) { @@ -102,7 +102,7 @@ return serverName.getServerName(); } @@ -358,7 +383,7 @@ index 56a0c8096d..2da74e9709 100644 ServerMetrics sl = master.getServerManager().getLoad(metaLocation); // The host name portion should be safe, but I don't know how we handle IDNs so err on the side of failing safely. - hostAndPort = URLEncoder.encode(metaLocation.getHostname()) + ":" + master.getRegionServerInfoPort(metaLocation); -+ hostAndPort = URLEncoder.encode(metaLocation.getExternalHostname()) + ":" + master.getRegionServerInfoPort(metaLocation); ++ hostAndPort = URLEncoder.encode(metaLocation.getExternalHostname()) + ":" + metaLocation.getExternalPort().toString(); if (sl != null) { Map map = sl.getRegionMetrics(); if (map.containsKey(meta.getRegionName())) { @@ -367,7 +392,7 @@ index 56a0c8096d..2da74e9709 100644 if (metaLocation != null) { ServerMetrics sl = master.getServerManager().getLoad(metaLocation); - hostAndPort = URLEncoder.encode(metaLocation.getHostname()) + ":" + master.getRegionServerInfoPort(metaLocation); -+ hostAndPort = URLEncoder.encode(metaLocation.getExternalHostname()) + ":" + master.getRegionServerInfoPort(metaLocation); ++ hostAndPort = URLEncoder.encode(metaLocation.getExternalHostname()) + ":" + metaLocation.getExternalPort().toString(); if (sl != null) { Map map = sl.getRegionMetrics(); if (map.containsKey(meta.getRegionName())) { @@ -376,16 +401,25 @@ index 56a0c8096d..2da74e9709 100644 if (metaLocation != null) { ServerMetrics sl = master.getServerManager().getLoad(metaLocation); - hostAndPort = URLEncoder.encode(metaLocation.getHostname()) + ":" + master.getRegionServerInfoPort(metaLocation); -+ hostAndPort = URLEncoder.encode(metaLocation.getExternalHostname()) + ":" + master.getRegionServerInfoPort(metaLocation); ++ hostAndPort = URLEncoder.encode(metaLocation.getExternalHostname()) + ":" + metaLocation.getExternalPort().toString(); if (sl != null) { Map map = sl.getRegionMetrics(); if (map.containsKey(meta.getRegionName())) { +@@ -548,7 +548,7 @@ + <%= endKeyDisplay %> + <%= replicaIdDisplay %> + <%= regionStateDisplay %> +- "><%= serverName != null ? buildRegionServerLink(serverName, master.getRegionServerInfoPort(serverName), regionInfo, regionState) : "" %> ++ "><%= serverName != null ? buildRegionServerLink(serverName, regionInfo, regionState) : "" %> + <%= seqNum %> + <%= targetServerName %> + <%= mergeRegionNames %> @@ -951,7 +951,7 @@ ServerMetrics sl = master.getServerManager().getLoad(addr); // This port might be wrong if RS actually ended up using something else. urlRegionServer = - "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; -+ "//" + URLEncoder.encode(addr.getExternalHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; ++ "//" + URLEncoder.encode(addr.getExternalHostname()) + ":" + addr.getExternalPort().toString() + "/rs-status"; if(sl != null) { Integer i = regDistribution.get(addr); if (null == i) i = Integer.valueOf(0); @@ -394,7 +428,7 @@ index 56a0c8096d..2da74e9709 100644 %> - <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> -+ <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getExternalHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> ++ <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getExternalHostname().toString()) + ":" + addr.getExternalPort().toString() %> <% } else { @@ -403,16 +437,25 @@ index 56a0c8096d..2da74e9709 100644 // This port might be wrong if RS actually ended up using something else. urlRegionServer = - "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; -+ "//" + URLEncoder.encode(addr.getExternalHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; ++ "//" + URLEncoder.encode(addr.getExternalHostname()) + ":" + addr.getExternalPort().toString() + "/rs-status"; } if (numRegionsRendered < numRegionsToRender) { numRegionsRendered++; +@@ -1048,7 +1048,7 @@ + if (urlRegionServer != null) { + %> + +- <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> ++ <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getExternalHostname().toString()) + ":" + addr.getExternalPort().toString() %> + + <% + } else { @@ -1098,7 +1098,7 @@ if (addr != null) { // This port might be wrong if RS actually ended up using something else. urlRegionServer = - "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; -+ "//" + URLEncoder.encode(addr.getExternalHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; ++ "//" + URLEncoder.encode(addr.getExternalHostname()) + ":" + addr.getExternalPort().toString() + "/rs-status"; } if (numRegionsRendered < numRegionsToRender) { numRegionsRendered++; @@ -421,7 +464,7 @@ index 56a0c8096d..2da74e9709 100644 %> - <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> -+ <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getExternalHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> ++ <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getExternalHostname().toString()) + ":" + addr.getExternalPort().toString() %> <% } else { @@ -430,11 +473,11 @@ index 56a0c8096d..2da74e9709 100644 for (Map.Entry rdEntry : regDistribution.entrySet()) { ServerName addr = rdEntry.getKey(); - String url = "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; -+ String url = "//" + URLEncoder.encode(addr.getExternalHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; ++ String url = "//" + URLEncoder.encode(addr.getExternalHostname()) + ":" + addr.getExternalPort().toString() + "/rs-status"; %> - <%= StringEscapeUtils.escapeHtml4(addr.getHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> -+ <%= StringEscapeUtils.escapeHtml4(addr.getExternalHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> ++ <%= StringEscapeUtils.escapeHtml4(addr.getExternalHostname().toString()) + ":" + addr.getExternalPort().toString() %> <%= rdEntry.getValue()%> <% if (withReplica) { From bd17a9ed4a4c0efb7f47c96f074f97bc0ee9ab9a Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 15 May 2025 14:55:44 +0200 Subject: [PATCH 05/12] rework patch for UI --- ...rvername-with-external-host-and-port.patch | 100 +++++++++++------- 1 file changed, 61 insertions(+), 39 deletions(-) diff --git a/hbase/stackable/patches/2.6.1/0005-extend-servername-with-external-host-and-port.patch b/hbase/stackable/patches/2.6.1/0005-extend-servername-with-external-host-and-port.patch index 763a7c5ac..b8c2e09b9 100644 --- a/hbase/stackable/patches/2.6.1/0005-extend-servername-with-external-host-and-port.patch +++ b/hbase/stackable/patches/2.6.1/0005-extend-servername-with-external-host-and-port.patch @@ -1,24 +1,25 @@ -From 6f9e3cae1bd74e7feeca428d34f078c5542dfc2c Mon Sep 17 00:00:00 2001 +From 958fd8f17dd203329b1da058e4714762412b4c57 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 16 Apr 2025 16:43:33 +0200 Subject: extend servername with external host and port --- - .../hbase/shaded/protobuf/ProtobufUtil.java | 15 ++++++-- - .../org/apache/hadoop/hbase/ServerName.java | 35 ++++++++++++++++++- - .../src/main/protobuf/HBase.proto | 2 ++ + .../hbase/shaded/protobuf/ProtobufUtil.java | 15 ++++++- + .../org/apache/hadoop/hbase/ServerName.java | 43 ++++++++++++++++++- + .../src/main/protobuf/HBase.proto | 2 + .../main/protobuf/RegionServerStatus.proto | 3 ++ - hbase-protocol/src/main/protobuf/HBase.proto | 2 ++ - .../src/main/protobuf/ZooKeeper.proto | 2 ++ + hbase-protocol/src/main/protobuf/HBase.proto | 2 + + .../src/main/protobuf/ZooKeeper.proto | 2 + .../tmpl/master/RegionServerListTmpl.jamon | 2 +- - .../apache/hadoop/hbase/master/HMaster.java | 7 ++++ - .../hadoop/hbase/master/ServerManager.java | 4 ++- - .../hbase/regionserver/HRegionServer.java | 16 ++++++++- - .../resources/hbase-webapps/master/hbck.jsp | 4 +-- - .../hbase-webapps/master/rsgroup.jsp | 15 +++----- - .../resources/hbase-webapps/master/table.jsp | 30 ++++++++-------- + .../tmpl/regionserver/RSStatusTmpl.jamon | 2 +- + .../apache/hadoop/hbase/master/HMaster.java | 7 +++ + .../hadoop/hbase/master/ServerManager.java | 4 +- + .../hbase/regionserver/HRegionServer.java | 16 ++++++- + .../resources/hbase-webapps/master/hbck.jsp | 4 +- + .../hbase-webapps/master/rsgroup.jsp | 15 +++---- + .../resources/hbase-webapps/master/table.jsp | 30 ++++++------- .../hbase/zookeeper/MasterAddressTracker.java | 3 ++ - 14 files changed, 107 insertions(+), 33 deletions(-) + 15 files changed, 116 insertions(+), 34 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java index 0fe181b503..be121ea641 100644 @@ -68,7 +69,7 @@ index 0fe181b503..be121ea641 100644 // A failed parse of the znode is pretty catastrophic. Rather than loop // retrying hoping the bad bytes will changes, and rather than change diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/ServerName.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/ServerName.java -index 5223bac3e5..932865ca69 100644 +index 5223bac3e5..979eb4c6ae 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/ServerName.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ServerName.java @@ -85,6 +85,8 @@ public class ServerName implements Comparable, Serializable { @@ -115,7 +116,7 @@ index 5223bac3e5..932865ca69 100644 /** * Retrieve an instance of ServerName. Callers should use the equals method to compare returned * instances, though we may return a shared immutable object as an internal optimization. -@@ -253,6 +270,22 @@ public class ServerName implements Comparable, Serializable { +@@ -253,6 +270,30 @@ public class ServerName implements Comparable, Serializable { return this.address.getPort(); } @@ -127,6 +128,14 @@ index 5223bac3e5..932865ca69 100644 + return this.externalPort; + } + ++ public String getDisplayHostname() { ++ return this.externalHostname != null && !this.externalHostname.isEmpty() ? this.externalHostname: this.getHostname(); ++ } ++ ++ public Integer getDisplayPort() { ++ return this.externalPort != null && this.externalPort > 0 ? this.externalPort: this.getPort(); ++ } ++ + public void setAddress(Address address) { + this.address = address; + } @@ -192,7 +201,7 @@ index c0f40e970e..bddadf4d30 100644 /** diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon -index 4c9140470b..604ecd7b2a 100644 +index 4c9140470b..641f558d87 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon @@ -500,7 +500,7 @@ if (totalCompactingCells > 0) { @@ -200,10 +209,23 @@ index 4c9140470b..604ecd7b2a 100644 <%java> int infoPort = master.getRegionServerInfoPort(serverName); - String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; -+ String url = "//" + serverName.getExternalHostname() + ":" + serverName.getExternalPort() + "/rs-status"; ++ String url = "//" + serverName.getDisplayHostname() + ":" + serverName.getDisplayPort() + "/rs-status"; <%if infoPort > 0%> +diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon +index 23330096cf..3abe5d617d 100644 +--- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon ++++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon +@@ -245,7 +245,7 @@ org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; + No master found + <%else> + <%java> +- String host = masterServerName.getHostname() + ":" + infoPort; ++ String host = masterServerName.getDisplayHostname() + ":" + masterServerName.getDisplayPort(); + String url = "//" + host + "/master-status"; + + <% host %> diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 3fe5abac27..652c709ab5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -279,7 +301,7 @@ index 27bcef2f06..8438378059 100644 } catch (ServiceException se) { IOException ioe = ProtobufUtil.getRemoteException(se); diff --git a/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp b/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp -index 38e16ca8e2..c30fd7753e 100644 +index 38e16ca8e2..61e4cf2b17 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp @@ -364,9 +364,9 @@ private static String formatServerName(HMaster master, @@ -287,15 +309,15 @@ index 38e16ca8e2..c30fd7753e 100644 String sn = serverName.toString(); if (serverManager.isServerOnline(serverName)) { - int infoPort = master.getRegionServerInfoPort(serverName); -+ int infoPort = serverName.getExternalPort(); ++ int infoPort = serverName.getDisplayPort(); if (infoPort > 0) { - return "" + sn + ""; } else { return "" + sn + ""; diff --git a/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp b/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp -index 6ba6f78a27..f3275b9165 100644 +index 6ba6f78a27..d05e1bc8f0 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp @@ -170,8 +170,7 @@ @@ -304,7 +326,7 @@ index 6ba6f78a27..f3275b9165 100644 long startcode = serverName.getStartcode(); - int infoPort = master.getRegionServerInfoPort(serverName); - String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status";%> -+ String url = "//" + serverName.getExternalHostname() + ":" + serverName.getExternalPort() + "/rs-status";%> ++ String url = "//" + serverName.getDisplayHostname() + ":" + serverName.getDisplayPort() + "/rs-status";%> <%= serverName.getServerName() %> <%= new Date(startcode) %> @@ -314,7 +336,7 @@ index 6ba6f78a27..f3275b9165 100644 .sum(); - int infoPort = master.getRegionServerInfoPort(serverName); - String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; -+ String url = "//" + serverName.getExternalHostname() + ":" + serverName.getExternalPort() + "/rs-status"; ++ String url = "//" + serverName.getDisplayHostname() + ":" + serverName.getDisplayPort() + "/rs-status"; if (memStoreSizeMB > 0) { memStoreSizeMBStr = TraditionalBinaryPrefix.long2String( @@ -330,7 +352,7 @@ index 6ba6f78a27..f3275b9165 100644 writeRequestCount += rm.getWriteRequestCount(); } - String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; -+ String url = "//" + serverName.getExternalHostname() + ":" + serverName.getExternalPort() + "/rs-status"; ++ String url = "//" + serverName.getDisplayHostname() + ":" + serverName.getDisplayPort() + "/rs-status"; %> <%= serverName.getServerName() %> @@ -340,7 +362,7 @@ index 6ba6f78a27..f3275b9165 100644 } - int infoPort = master.getRegionServerInfoPort(serverName); - String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; -+ String url = "//" + serverName.getExternalHostname() + ":" + serverName.getExternalPort() + "/rs-status"; ++ String url = "//" + serverName.getDisplayHostname() + ":" + serverName.getDisplayPort() + "/rs-status"; if (storeUncompressedSizeMB > 0) { storeUncompressedSizeMBStr = TraditionalBinaryPrefix.long2String( (long) storeUncompressedSizeMB * TraditionalBinaryPrefix.MEGA.value, "B", 1); @@ -350,12 +372,12 @@ index 6ba6f78a27..f3275b9165 100644 } - int infoPort = master.getRegionServerInfoPort(serverName); - String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; -+ String url = "//" + serverName.getExternalHostname() + ":" + serverName.getExternalPort() + "/rs-status"; ++ String url = "//" + serverName.getDisplayHostname() + ":" + serverName.getDisplayPort() + "/rs-status"; %> <%= serverName.getServerName() %> diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp -index 56a0c8096d..49c8a69c60 100644 +index 56a0c8096d..01fbde1b18 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp @@ -93,8 +93,8 @@ @@ -374,7 +396,7 @@ index 56a0c8096d..49c8a69c60 100644 } - final String socketAddress = serverName.getHostname() + ":" + rsInfoPort; -+ final String socketAddress = serverName.getExternalHostname() + ":" + serverName.getExternalPort(); ++ final String socketAddress = serverName.getDisplayHostname() + ":" + serverName.getDisplayPort(); final String URI = "//" + socketAddress + "/region.jsp" + "?name=" + regionInfo.getEncodedName(); return "" + serverName.getServerName() + ""; @@ -383,7 +405,7 @@ index 56a0c8096d..49c8a69c60 100644 ServerMetrics sl = master.getServerManager().getLoad(metaLocation); // The host name portion should be safe, but I don't know how we handle IDNs so err on the side of failing safely. - hostAndPort = URLEncoder.encode(metaLocation.getHostname()) + ":" + master.getRegionServerInfoPort(metaLocation); -+ hostAndPort = URLEncoder.encode(metaLocation.getExternalHostname()) + ":" + metaLocation.getExternalPort().toString(); ++ hostAndPort = URLEncoder.encode(metaLocation.getDisplayHostname()) + ":" + metaLocation.getDisplayPort().toString(); if (sl != null) { Map map = sl.getRegionMetrics(); if (map.containsKey(meta.getRegionName())) { @@ -392,7 +414,7 @@ index 56a0c8096d..49c8a69c60 100644 if (metaLocation != null) { ServerMetrics sl = master.getServerManager().getLoad(metaLocation); - hostAndPort = URLEncoder.encode(metaLocation.getHostname()) + ":" + master.getRegionServerInfoPort(metaLocation); -+ hostAndPort = URLEncoder.encode(metaLocation.getExternalHostname()) + ":" + metaLocation.getExternalPort().toString(); ++ hostAndPort = URLEncoder.encode(metaLocation.getDisplayHostname()) + ":" + metaLocation.getDisplayPort().toString(); if (sl != null) { Map map = sl.getRegionMetrics(); if (map.containsKey(meta.getRegionName())) { @@ -401,7 +423,7 @@ index 56a0c8096d..49c8a69c60 100644 if (metaLocation != null) { ServerMetrics sl = master.getServerManager().getLoad(metaLocation); - hostAndPort = URLEncoder.encode(metaLocation.getHostname()) + ":" + master.getRegionServerInfoPort(metaLocation); -+ hostAndPort = URLEncoder.encode(metaLocation.getExternalHostname()) + ":" + metaLocation.getExternalPort().toString(); ++ hostAndPort = URLEncoder.encode(metaLocation.getDisplayHostname()) + ":" + metaLocation.getDisplayPort().toString(); if (sl != null) { Map map = sl.getRegionMetrics(); if (map.containsKey(meta.getRegionName())) { @@ -419,7 +441,7 @@ index 56a0c8096d..49c8a69c60 100644 // This port might be wrong if RS actually ended up using something else. urlRegionServer = - "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; -+ "//" + URLEncoder.encode(addr.getExternalHostname()) + ":" + addr.getExternalPort().toString() + "/rs-status"; ++ "//" + URLEncoder.encode(addr.getDisplayHostname()) + ":" + addr.getDisplayPort().toString() + "/rs-status"; if(sl != null) { Integer i = regDistribution.get(addr); if (null == i) i = Integer.valueOf(0); @@ -428,7 +450,7 @@ index 56a0c8096d..49c8a69c60 100644 %> - <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> -+ <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getExternalHostname().toString()) + ":" + addr.getExternalPort().toString() %> ++ <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getDisplayHostname().toString()) + ":" + addr.getDisplayPort().toString() %> <% } else { @@ -437,7 +459,7 @@ index 56a0c8096d..49c8a69c60 100644 // This port might be wrong if RS actually ended up using something else. urlRegionServer = - "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; -+ "//" + URLEncoder.encode(addr.getExternalHostname()) + ":" + addr.getExternalPort().toString() + "/rs-status"; ++ "//" + URLEncoder.encode(addr.getDisplayHostname()) + ":" + addr.getDisplayPort().toString() + "/rs-status"; } if (numRegionsRendered < numRegionsToRender) { numRegionsRendered++; @@ -446,7 +468,7 @@ index 56a0c8096d..49c8a69c60 100644 %> - <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> -+ <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getExternalHostname().toString()) + ":" + addr.getExternalPort().toString() %> ++ <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getDisplayHostname().toString()) + ":" + addr.getDisplayPort().toString() %> <% } else { @@ -455,7 +477,7 @@ index 56a0c8096d..49c8a69c60 100644 // This port might be wrong if RS actually ended up using something else. urlRegionServer = - "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; -+ "//" + URLEncoder.encode(addr.getExternalHostname()) + ":" + addr.getExternalPort().toString() + "/rs-status"; ++ "//" + URLEncoder.encode(addr.getDisplayHostname()) + ":" + addr.getDisplayPort().toString() + "/rs-status"; } if (numRegionsRendered < numRegionsToRender) { numRegionsRendered++; @@ -464,7 +486,7 @@ index 56a0c8096d..49c8a69c60 100644 %> - <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> -+ <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getExternalHostname().toString()) + ":" + addr.getExternalPort().toString() %> ++ <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getDisplayHostname().toString()) + ":" + addr.getDisplayPort().toString() %> <% } else { @@ -473,11 +495,11 @@ index 56a0c8096d..49c8a69c60 100644 for (Map.Entry rdEntry : regDistribution.entrySet()) { ServerName addr = rdEntry.getKey(); - String url = "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; -+ String url = "//" + URLEncoder.encode(addr.getExternalHostname()) + ":" + addr.getExternalPort().toString() + "/rs-status"; ++ String url = "//" + URLEncoder.encode(addr.getDisplayHostname()) + ":" + addr.getDisplayPort().toString() + "/rs-status"; %> - <%= StringEscapeUtils.escapeHtml4(addr.getHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> -+ <%= StringEscapeUtils.escapeHtml4(addr.getExternalHostname().toString()) + ":" + addr.getExternalPort().toString() %> ++ <%= StringEscapeUtils.escapeHtml4(addr.getDisplayHostname().toString()) + ":" + addr.getDisplayPort().toString() %> <%= rdEntry.getValue()%> <% if (withReplica) { From 469cd77eeee8682f8016b94d4e620624b223e02a Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 15 May 2025 15:07:57 +0200 Subject: [PATCH 06/12] wip --- airflow/Dockerfile | 2 +- airflow/constraints-3.0.1-python3.12.txt | 813 +++++++++++++++++++++++ airflow/versions.py | 8 + 3 files changed, 822 insertions(+), 1 deletion(-) create mode 100644 airflow/constraints-3.0.1-python3.12.txt diff --git a/airflow/Dockerfile b/airflow/Dockerfile index 951f07ea9..6d643f9d5 100644 --- a/airflow/Dockerfile +++ b/airflow/Dockerfile @@ -35,7 +35,7 @@ COPY --from=opa-auth-manager-builder /tmp/opa-auth-manager/dist/opa_auth_manager # The mysql provider is currently excluded. # Requires implementation of https://github.com/apache/airflow/blob/2.2.5/scripts/docker/install_mysql.sh -ENV AIRFLOW_EXTRAS=async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,ldap,google,google_auth,microsoft.azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,trino +ENV AIRFLOW_EXTRAS=async,amazon,celery,cncf-kubernetes,docker,elasticsearch,fab,ftp,grpc,hashicorp,http,ldap,google,microsoft-azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,trino RUN microdnf update && \ microdnf install \ diff --git a/airflow/constraints-3.0.1-python3.12.txt b/airflow/constraints-3.0.1-python3.12.txt new file mode 100644 index 000000000..bf304ec8d --- /dev/null +++ b/airflow/constraints-3.0.1-python3.12.txt @@ -0,0 +1,813 @@ + +# +# This constraints file was automatically generated on 2025-05-10T16:44:38.068881 +# via `uv pip install --resolution highest` for the "v3-0-test" branch of Airflow. +# This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs +# the providers from PIP-released packages at the moment of the constraint generation. +# +# Those constraints are actually those that regular users use to install released version of Airflow. +# We also use those constraints after "apache-airflow" is released and the constraints are tagged with +# "constraints-X.Y.Z" tag to build the production image for that version. +# +# This constraints file is meant to be used only in the "apache-airflow" installation command and not +# in all subsequent pip commands. By using a constraints.txt file, we ensure that solely the Airflow +# installation step is reproducible. Subsequent pip commands may install packages that would have +# been incompatible with the constraints used in Airflow reproducible installation step. Finally, pip +# commands that might change the installed version of apache-airflow should include "apache-airflow==X.Y.Z" +# in the list of install targets to prevent Airflow accidental upgrade or downgrade. +# +# Typical installation process of airflow for Python 3.9 is (with random selection of extras and custom +# dependencies added), usually consists of two steps: +# +# 1. Reproducible installation of airflow with selected providers (note constraints are used): +# +# pip install "apache-airflow[celery,cncf.kubernetes,google,amazon,snowflake]==X.Y.Z" \ +# --constraint \ +# "https://raw.githubusercontent.com/apache/airflow/constraints-X.Y.Z/constraints-3.12.txt" +# +# 2. Installing own dependencies that are potentially not matching the constraints (note constraints are not +# used, and apache-airflow==X.Y.Z is used to make sure there is no accidental airflow upgrade/downgrade. +# +# pip install "apache-airflow==X.Y.Z" "snowflake-connector-python[pandas]=N.M.O" +# +APScheduler==3.11.0 +Authlib==1.3.1 +Deprecated==1.2.18 +Events==0.5 +Flask-AppBuilder==4.5.3 +Flask-Babel==2.0.0 +Flask-JWT-Extended==4.7.1 +Flask-Limiter==3.12 +Flask-Login==0.6.3 +Flask-SQLAlchemy==2.5.1 +Flask-Session==0.5.0 +Flask-WTF==1.2.2 +Flask==2.2.5 +GitPython==3.1.44 +JayDeBeApi==1.2.3 +Jinja2==3.1.6 +Mako==1.3.10 +Markdown==3.8 +MarkupSafe==3.0.2 +PyAthena==3.13.0 +PyGithub==2.6.1 +PyHive==0.7.0 +PyJWT==2.10.1 +PyMySQL==1.1.1 +PyNaCl==1.5.0 +PyYAML==6.0.2 +Pygments==2.19.1 +SQLAlchemy-JSONField==1.0.2 +SQLAlchemy-Utils==0.41.2 +SQLAlchemy==1.4.54 +SecretStorage==3.3.3 +Sphinx==8.2.3 +WTForms==3.2.1 +Werkzeug==2.2.3 +a2wsgi==1.10.8 +adal==1.2.7 +adlfs==2024.12.0 +aiobotocore==2.22.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp-cors==0.8.1 +aiohttp==3.11.18 +aioitertools==0.12.0 +aiologic==0.14.0 +aiomysql==0.2.0 +aioresponses==0.7.8 +aiosignal==1.3.2 +aiosqlite==0.21.0 +airbyte-api==0.52.2 +alabaster==1.0.0 +alembic==1.15.2 +alibabacloud-adb20211201==3.0.3 +alibabacloud-credentials-api==1.0.0 +alibabacloud-credentials==1.0.2 +alibabacloud-tea==0.4.3 +alibabacloud_endpoint_util==0.0.3 +alibabacloud_gateway_spi==0.0.3 +alibabacloud_openapi_util==0.2.2 +alibabacloud_tea_openapi==0.3.15 +alibabacloud_tea_util==0.3.13 +alibabacloud_tea_xml==0.0.2 +aliyun-python-sdk-core==2.16.0 +aliyun-python-sdk-kms==2.16.5 +amqp==5.3.1 +analytics-python==1.2.9 +annotated-types==0.7.0 +ansicolors==1.1.8 +anyio==4.9.0 +apache-airflow-providers-airbyte==5.0.2 +apache-airflow-providers-alibaba==3.0.3 +apache-airflow-providers-amazon==9.7.0 +apache-airflow-providers-apache-beam==6.0.4 +apache-airflow-providers-apache-cassandra==3.7.2 +apache-airflow-providers-apache-drill==3.0.1 +apache-airflow-providers-apache-druid==4.1.1 +apache-airflow-providers-apache-flink==1.6.2 +apache-airflow-providers-apache-hdfs==4.8.1 +apache-airflow-providers-apache-hive==9.0.6 +apache-airflow-providers-apache-iceberg==1.2.1 +apache-airflow-providers-apache-impala==1.6.1 +apache-airflow-providers-apache-kafka==1.8.1 +apache-airflow-providers-apache-kylin==3.8.3 +apache-airflow-providers-apache-livy==4.3.2 +apache-airflow-providers-apache-pig==4.6.1 +apache-airflow-providers-apache-pinot==4.7.0 +apache-airflow-providers-apache-spark==5.2.1 +apache-airflow-providers-apprise==2.0.2 +apache-airflow-providers-arangodb==2.7.4 +apache-airflow-providers-asana==2.9.1 +apache-airflow-providers-atlassian-jira==3.0.2 +apache-airflow-providers-celery==3.10.6 +apache-airflow-providers-cloudant==4.1.1 +apache-airflow-providers-cncf-kubernetes==10.4.3 +apache-airflow-providers-cohere==1.4.3 +apache-airflow-providers-common-compat==1.6.1 +apache-airflow-providers-common-io==1.5.4 +apache-airflow-providers-common-messaging==1.0.1 +apache-airflow-providers-common-sql==1.27.0 +apache-airflow-providers-databricks==7.3.2 +apache-airflow-providers-datadog==3.8.3 +apache-airflow-providers-dbt-cloud==4.3.3 +apache-airflow-providers-dingding==3.7.3 +apache-airflow-providers-discord==3.9.5 +apache-airflow-providers-docker==4.3.1 +apache-airflow-providers-edge3==1.0.0 +apache-airflow-providers-elasticsearch==6.2.2 +apache-airflow-providers-exasol==4.7.5 +apache-airflow-providers-fab==2.0.2 +apache-airflow-providers-facebook==3.7.2 +apache-airflow-providers-ftp==3.12.3 +apache-airflow-providers-git==0.0.2 +apache-airflow-providers-github==2.8.3 +apache-airflow-providers-google==15.1.0 +apache-airflow-providers-grpc==3.7.3 +apache-airflow-providers-hashicorp==4.1.1 +apache-airflow-providers-http==5.2.2 +apache-airflow-providers-imap==3.8.3 +apache-airflow-providers-influxdb==2.8.3 +apache-airflow-providers-jdbc==5.1.0 +apache-airflow-providers-jenkins==4.0.4 +apache-airflow-providers-microsoft-azure==12.3.1 +apache-airflow-providers-microsoft-mssql==4.2.2 +apache-airflow-providers-microsoft-psrp==3.0.1 +apache-airflow-providers-microsoft-winrm==3.9.2 +apache-airflow-providers-mongo==5.0.3 +apache-airflow-providers-mysql==6.2.2 +apache-airflow-providers-neo4j==3.8.2 +apache-airflow-providers-odbc==4.9.2 +apache-airflow-providers-openai==1.5.4 +apache-airflow-providers-openfaas==3.7.2 +apache-airflow-providers-openlineage==2.2.0 +apache-airflow-providers-opensearch==1.6.3 +apache-airflow-providers-opsgenie==5.8.3 +apache-airflow-providers-oracle==4.0.3 +apache-airflow-providers-pagerduty==4.0.3 +apache-airflow-providers-papermill==3.10.1 +apache-airflow-providers-pgvector==1.4.1 +apache-airflow-providers-pinecone==2.2.2 +apache-airflow-providers-postgres==6.1.3 +apache-airflow-providers-presto==5.8.4 +apache-airflow-providers-qdrant==1.3.2 +apache-airflow-providers-redis==4.0.2 +apache-airflow-providers-salesforce==5.10.1 +apache-airflow-providers-samba==4.9.2 +apache-airflow-providers-segment==3.7.2 +apache-airflow-providers-sendgrid==4.0.1 +apache-airflow-providers-sftp==5.2.1 +apache-airflow-providers-singularity==3.7.1 +apache-airflow-providers-slack==9.0.5 +apache-airflow-providers-smtp==2.0.3 +apache-airflow-providers-snowflake==6.3.0 +apache-airflow-providers-sqlite==4.0.2 +apache-airflow-providers-ssh==4.0.1 +apache-airflow-providers-standard==1.1.0 +apache-airflow-providers-tableau==5.0.2 +apache-airflow-providers-telegram==4.7.2 +apache-airflow-providers-teradata==3.0.3 +apache-airflow-providers-trino==6.1.2 +apache-airflow-providers-vertica==4.0.1 +apache-airflow-providers-weaviate==3.0.3 +apache-airflow-providers-yandex==4.0.3 +apache-airflow-providers-ydb==2.1.1 +apache-airflow-providers-zendesk==4.9.1 +apache-beam==2.59.0 +apispec==6.8.1 +apprise==1.9.3 +argcomplete==3.6.2 +asana==5.1.0 +asgiref==3.8.1 +asn1crypto==1.5.1 +astroid==3.3.10 +asttokens==3.0.0 +asyncpg==0.30.0 +asyncssh==2.21.0 +atlasclient==1.0.0 +atlassian-python-api==4.0.3 +attrs==25.3.0 +aws-sam-translator==1.97.0 +aws-xray-sdk==2.14.0 +azure-batch==14.2.0 +azure-common==1.1.28 +azure-core==1.34.0 +azure-cosmos==4.9.0 +azure-datalake-store==0.0.53 +azure-identity==1.22.0 +azure-keyvault-secrets==4.9.0 +azure-kusto-data==5.0.3 +azure-mgmt-containerinstance==10.1.0 +azure-mgmt-containerregistry==14.0.0 +azure-mgmt-core==1.5.0 +azure-mgmt-cosmosdb==9.8.0 +azure-mgmt-datafactory==9.2.0 +azure-mgmt-datalake-nspkg==3.0.1 +azure-mgmt-datalake-store==0.5.0 +azure-mgmt-nspkg==3.0.2 +azure-mgmt-resource==23.3.0 +azure-mgmt-storage==22.2.0 +azure-nspkg==3.0.2 +azure-servicebus==7.14.2 +azure-storage-blob==12.25.1 +azure-storage-file-datalake==12.20.0 +azure-storage-file-share==12.21.0 +azure-synapse-artifacts==0.20.0 +azure-synapse-spark==0.7.0 +babel==2.17.0 +backoff==2.2.1 +bcrypt==4.3.0 +beautifulsoup4==4.13.4 +billiard==4.2.1 +bitarray==2.9.3 +black==25.1.0 +bleach==6.2.0 +blinker==1.9.0 +boto3==1.37.3 +botocore==1.37.3 +cachelib==0.13.0 +cachetools==5.5.2 +cadwyn==5.3.3 +cassandra-driver==3.29.2 +cattrs==24.1.3 +celery==5.5.2 +certifi==2025.4.26 +cffi==1.17.1 +cfgv==3.4.0 +cfn-lint==1.35.1 +chardet==5.2.0 +charset-normalizer==3.4.2 +checksumdir==1.2.0 +ciso8601==2.3.2 +click-didyoumean==0.3.1 +click-plugins==1.1.1 +click-repl==0.3.0 +click==8.1.8 +clickclick==20.10.2 +cloudpickle==2.2.1 +cohere==5.15.0 +colorama==0.4.6 +colorful==0.5.6 +colorlog==6.9.0 +comm==0.2.2 +confluent-kafka==2.10.0 +connexion==2.14.2 +coverage==7.8.0 +crcmod==1.7 +cron-descriptor==1.4.5 +croniter==6.0.0 +cryptography==42.0.8 +curlify==2.2.1 +databricks-sql-connector==4.0.3 +dataclasses-json==0.6.7 +datadog==0.51.0 +db-dtypes==1.4.2 +debugpy==1.8.14 +decorator==5.2.1 +defusedxml==0.7.1 +deltalake==0.25.5 +diagrams==0.24.4 +dill==0.3.1.1 +distlib==0.3.9 +distro==1.9.0 +dnspython==2.7.0 +docker==7.1.0 +docopt==0.6.2 +docstring_parser==0.16 +docutils==0.21.2 +duckdb==1.2.2 +durationpy==0.9 +elastic-transport==8.17.1 +elasticsearch==8.18.1 +email_validator==2.2.0 +entrypoints==0.4 +eralchemy2==1.4.1 +et_xmlfile==2.0.0 +eventlet==0.39.1 +execnet==2.1.1 +executing==2.2.0 +facebook_business==22.0.5 +fastapi-cli==0.0.7 +fastapi==0.115.12 +fastavro==1.10.0 +fasteners==0.19 +fastjsonschema==2.21.1 +filelock==3.18.0 +flit==3.12.0 +flit_core==3.12.0 +flower==2.0.1 +frozenlist==1.6.0 +fsspec==2025.3.2 +future==1.0.0 +gcloud-aio-auth==5.4.1 +gcloud-aio-bigquery==7.1.0 +gcloud-aio-storage==9.4.0 +gcsfs==2025.3.2 +geomet==0.2.1.post1 +gevent==25.4.2 +gitdb==4.0.12 +google-ads==26.1.0 +google-analytics-admin==0.24.0 +google-api-core==2.24.2 +google-api-python-client==2.169.0 +google-auth-httplib2==0.2.0 +google-auth-oauthlib==1.2.2 +google-auth==2.40.1 +google-cloud-aiplatform==1.92.0 +google-cloud-alloydb==0.4.5 +google-cloud-appengine-logging==1.6.1 +google-cloud-audit-log==0.3.2 +google-cloud-automl==2.16.3 +google-cloud-batch==0.17.35 +google-cloud-bigquery-datatransfer==3.19.1 +google-cloud-bigquery-storage==2.31.0 +google-cloud-bigquery==3.31.0 +google-cloud-bigtable==2.30.1 +google-cloud-build==3.31.1 +google-cloud-compute==1.30.0 +google-cloud-container==2.56.1 +google-cloud-core==2.4.3 +google-cloud-datacatalog==3.27.1 +google-cloud-dataflow-client==0.9.0 +google-cloud-dataform==0.6.1 +google-cloud-dataplex==2.10.1 +google-cloud-dataproc-metastore==1.18.2 +google-cloud-dataproc==5.18.1 +google-cloud-dlp==3.29.0 +google-cloud-kms==3.4.1 +google-cloud-language==2.17.1 +google-cloud-logging==3.12.1 +google-cloud-managedkafka==0.1.10 +google-cloud-memcache==1.12.1 +google-cloud-monitoring==2.27.1 +google-cloud-orchestration-airflow==1.17.5 +google-cloud-os-login==2.17.1 +google-cloud-pubsub==2.29.0 +google-cloud-redis==2.18.1 +google-cloud-resource-manager==1.14.2 +google-cloud-run==0.10.17 +google-cloud-secret-manager==2.23.3 +google-cloud-spanner==3.54.0 +google-cloud-speech==2.32.0 +google-cloud-storage-transfer==1.16.1 +google-cloud-storage==2.19.0 +google-cloud-tasks==2.19.2 +google-cloud-texttospeech==2.27.0 +google-cloud-translate==3.20.2 +google-cloud-videointelligence==2.16.1 +google-cloud-vision==3.10.1 +google-cloud-workflows==1.18.1 +google-crc32c==1.7.1 +google-genai==1.2.0 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +graphql-core==3.2.6 +graphviz==0.20.3 +greenlet==3.2.2 +grpc-google-iam-v1==0.14.2 +grpc-interceptor==0.15.4 +grpcio-gcp==0.2.2 +grpcio-health-checking==1.62.3 +grpcio-status==1.62.3 +grpcio-tools==1.62.3 +grpcio==1.71.0 +gssapi==1.9.0 +gunicorn==23.0.0 +h11==0.16.0 +h2==4.2.0 +hatch==1.14.1 +hatchling==1.27.0 +hdfs==2.7.3 +hf-xet==1.1.0 +hmsclient==0.1.1 +hpack==4.1.0 +httpcore==1.0.9 +httplib2==0.22.0 +httptools==0.6.4 +httpx-sse==0.4.0 +httpx==0.27.0 +huggingface-hub==0.31.1 +humanize==4.12.3 +hvac==2.3.0 +hyperframe==6.1.0 +hyperlink==21.0.0 +ibm-cloud-sdk-core==3.20.3 +ibmcloudant==0.9.1 +icdiff==2.0.7 +id==1.5.0 +identify==2.6.10 +idna==3.10 +ijson==3.4.0 +imagesize==1.4.1 +immutabledict==4.2.1 +importlib_metadata==8.4.0 +impyla==0.21.0 +incremental==24.7.2 +inflection==0.5.1 +influxdb-client==1.48.0 +iniconfig==2.1.0 +inputimeout==1.0.4 +ipdb==0.13.13 +ipykernel==6.29.5 +ipython==9.2.0 +ipython_pygments_lexers==1.1.1 +isodate==0.7.2 +itsdangerous==2.2.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.1.0 +jedi==0.19.2 +jeepney==0.9.0 +jiter==0.9.0 +jmespath==0.10.0 +joblib==1.5.0 +joserfc==1.0.4 +jpype1==1.5.2 +jsonpatch==1.33 +jsonpath-ng==1.7.0 +jsonpath-python==1.0.6 +jsonpickle==3.4.2 +jsonpointer==3.0.0 +jsonschema-path==0.3.4 +jsonschema-specifications==2025.4.1 +jsonschema==4.23.0 +jupyter_client==8.6.3 +jupyter_core==5.7.2 +jupyterlab_pygments==0.3.0 +kerberos==1.3.1 +keyring==25.6.0 +keyrings.alt==5.0.2 +kgb==7.2 +kombu==5.5.3 +krb5==0.7.1 +kubernetes==31.0.0 +kubernetes_asyncio==30.3.1 +kylinpy==2.8.4 +lazy-object-proxy==1.11.0 +libcst==1.7.0 +limits==5.1.0 +linkify-it-py==2.0.3 +lockfile==0.12.2 +looker-sdk==25.6.0 +lxml==5.3.2 +lz4==4.4.4 +markdown-it-py==3.0.0 +marshmallow-sqlalchemy==0.28.2 +marshmallow==3.26.1 +matplotlib-inline==0.1.7 +mdurl==0.1.2 +mergedeep==1.3.4 +methodtools==0.4.7 +microsoft-kiota-abstractions==1.9.3 +microsoft-kiota-authentication-azure==1.9.3 +microsoft-kiota-http==1.9.3 +microsoft-kiota-serialization-json==1.9.3 +microsoft-kiota-serialization-text==1.9.3 +mistune==3.1.3 +mmh3==5.1.0 +mongomock==4.3.0 +more-itertools==10.7.0 +moto==5.1.4 +mpmath==1.3.0 +msal-extensions==1.3.1 +msal==1.32.3 +msgpack==1.1.0 +msgraph-core==1.3.3 +msgspec==0.19.0 +msrest==0.7.1 +msrestazure==0.6.4.post1 +multi_key_dict==2.0.3 +multidict==6.4.3 +mypy-boto3-appflow==1.38.0 +mypy-boto3-rds==1.38.2 +mypy-boto3-redshift-data==1.38.0 +mypy-boto3-s3==1.38.0 +mypy==1.9.0 +mypy_extensions==1.1.0 +mysql-connector-python==9.3.0 +mysqlclient==2.2.7 +nbclient==0.10.2 +nbconvert==7.16.6 +nbformat==5.10.4 +neo4j==5.28.1 +nest-asyncio==1.6.0 +networkx==3.4.2 +nh3==0.2.21 +nodeenv==1.9.1 +numpy==1.26.4 +oauthlib==3.2.2 +objsize==0.7.1 +openai==1.78.0 +openapi-schema-validator==0.6.3 +openapi-spec-validator==0.7.1 +opencensus-context==0.1.3 +opencensus==0.11.4 +openlineage-integration-common==1.32.1 +openlineage-python==1.32.1 +openlineage_sql==1.32.1 +openpyxl==3.1.5 +opensearch-py==2.8.0 +opentelemetry-api==1.27.0 +opentelemetry-exporter-otlp-proto-common==1.27.0 +opentelemetry-exporter-otlp-proto-grpc==1.27.0 +opentelemetry-exporter-otlp-proto-http==1.27.0 +opentelemetry-exporter-otlp==1.27.0 +opentelemetry-exporter-prometheus==0.48b0 +opentelemetry-proto==1.27.0 +opentelemetry-sdk==1.27.0 +opentelemetry-semantic-conventions==0.48b0 +opsgenie-sdk==2.1.5 +oracledb==3.1.0 +ordered-set==4.1.0 +orjson==3.10.18 +oss2==2.19.1 +packaging==25.0 +pandas-gbq==0.28.0 +pandas-stubs==2.2.3.250308 +pandas==2.1.4 +pandocfilters==1.5.1 +papermill==2.6.0 +paramiko==3.5.1 +parso==0.8.4 +pathable==0.4.4 +pathspec==0.12.1 +pbr==6.1.1 +pdbr==0.9.0 +pdpyras==5.4.0 +pendulum==3.1.0 +pexpect==4.9.0 +pgvector==0.4.1 +pinecone-plugin-interface==0.0.7 +pinecone==6.0.2 +pinotdb==5.6.0 +pipdeptree==2.26.1 +platformdirs==4.3.8 +pluggy==1.5.0 +ply==3.11 +plyvel==1.5.1 +polars==1.29.0 +portalocker==2.10.1 +pprintpp==0.4.0 +pre-commit-uv==4.1.4 +pre_commit==4.2.0 +presto-python-client==0.8.4 +prison==0.2.1 +prometheus_client==0.21.1 +prompt_toolkit==3.0.51 +propcache==0.3.1 +proto-plus==1.26.1 +protobuf==4.25.7 +psutil==7.0.0 +psycopg2-binary==2.9.10 +ptyprocess==0.7.0 +pure-sasl==0.6.2 +pure_eval==0.2.3 +py-partiql-parser==0.6.1 +py-spy==0.4.0 +py4j==0.10.9.7 +pyOpenSSL==25.0.0 +pyarrow-hotfix==0.7 +pyarrow==16.1.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.1 +pycountry==24.6.1 +pycparser==2.22 +pycryptodome==3.22.0 +pydantic==2.11.4 +pydantic_core==2.33.2 +pydata-google-auth==1.9.1 +pydot==1.4.2 +pydruid==0.6.9 +pyenchant==3.2.2 +pyexasol==0.27.0 +pygraphviz==1.14 +pyiceberg==0.9.1 +pykerberos==1.2.4 +pymongo==4.10.1 +pymssql==2.3.4 +pyodbc==5.2.0 +pyparsing==3.2.3 +pypsrp==0.8.1 +pyspark==3.5.5 +pyspnego==0.11.2 +pytest-asyncio==0.25.0 +pytest-cov==6.1.1 +pytest-custom-exit-code==0.3.0 +pytest-icdiff==0.9 +pytest-instafail==0.5.0 +pytest-mock==3.14.0 +pytest-rerunfailures==15.1 +pytest-timeouts==1.2.1 +pytest-unordered==0.6.1 +pytest-xdist==3.6.1 +pytest==8.3.5 +python-arango==8.1.6 +python-daemon==3.1.2 +python-dateutil==2.9.0.post0 +python-dotenv==1.1.0 +python-http-client==3.3.7 +python-jenkins==1.8.2 +python-ldap==3.4.4 +python-multipart==0.0.20 +python-on-whales==0.76.1 +python-slugify==8.0.4 +python-telegram-bot==22.0 +python3-saml==1.16.0 +pytz==2025.2 +pywinrm==0.5.0 +pyzmq==26.4.0 +qdrant-client==1.14.2 +ray==2.46.0 +reactivex==4.0.4 +readme_renderer==44.0 +redis==5.2.1 +redshift-connector==2.1.6 +referencing==0.36.2 +regex==2024.11.6 +requests-file==2.1.0 +requests-kerberos==0.15.0 +requests-mock==1.12.1 +requests-oauthlib==2.0.0 +requests-toolbelt==1.0.0 +requests==2.32.3 +requests_ntlm==1.3.0 +responses==0.25.7 +restructuredtext_lint==1.4.0 +retryhttp==1.3.3 +rfc3339-validator==0.1.4 +rfc3986==2.0.0 +rich-argparse==1.7.0 +rich-click==1.8.8 +rich-toolkit==0.14.5 +rich==13.9.4 +roman-numerals-py==3.1.0 +rpds-py==0.24.0 +rsa==4.9.1 +ruamel.yaml.clib==0.2.12 +ruamel.yaml==0.18.10 +ruff==0.11.2 +s3fs==2025.3.2 +s3transfer==0.11.3 +sagemaker_studio==1.0.14 +scikit-learn==1.6.1 +scipy==1.15.3 +scramp==1.4.5 +scrapbook==0.5.0 +semver==3.0.4 +sendgrid==6.11.0 +sentinels==1.0.0 +sentry-sdk==2.27.0 +setproctitle==1.3.6 +setuptools==80.4.0 +shapely==2.1.0 +shellingham==1.5.4 +simple-salesforce==1.12.6 +six==1.17.0 +slack_sdk==3.35.0 +smart-open==7.1.0 +smbprotocol==1.15.0 +smmap==5.0.2 +sniffio==1.3.1 +snowballstemmer==3.0.1 +snowflake-connector-python==3.15.0 +snowflake-sqlalchemy==1.7.3 +sortedcontainers==2.4.0 +soupsieve==2.7 +sphinx-argparse==0.5.2 +sphinx-autoapi==3.6.0 +sphinx-autobuild==2024.10.3 +sphinx-copybutton==0.5.2 +sphinx-jinja==2.0.2 +sphinx-rtd-theme==3.0.2 +sphinx_airflow_theme==0.2.2 +sphinx_design==0.6.1 +sphinxcontrib-applehelp==2.0.0 +sphinxcontrib-devhelp==2.0.0 +sphinxcontrib-htmlhelp==2.1.0 +sphinxcontrib-httpdomain==1.8.1 +sphinxcontrib-jquery==4.1 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==2.0.0 +sphinxcontrib-redoc==1.6.0 +sphinxcontrib-serializinghtml==2.0.0 +sphinxcontrib-spelling==8.0.1 +spython==0.3.14 +sqlalchemy-bigquery==1.14.0 +sqlalchemy-spanner==1.11.0 +sqlalchemy_drill==1.1.8 +sqlparse==0.5.3 +sshtunnel==0.4.0 +stack-data==0.6.3 +starkbank-ecdsa==2.2.0 +starlette==0.46.2 +statsd==4.0.1 +std-uritemplate==2.0.3 +strictyaml==1.7.3 +structlog==25.3.0 +svcs==25.1.0 +sympy==1.14.0 +tableauserverclient==0.37 +tabulate==0.9.0 +tenacity==9.1.2 +teradatasql==20.0.0.30 +teradatasqlalchemy==20.0.0.4 +termcolor==3.1.0 +text-unidecode==1.3 +threadpoolctl==3.6.0 +thrift-sasl==0.4.3 +thrift==0.16.0 +time-machine==2.16.0 +tinycss2==1.4.0 +tokenizers==0.21.1 +tomli_w==1.2.0 +tomlkit==0.13.2 +tornado==6.4.2 +towncrier==24.8.0 +tqdm==4.67.1 +traitlets==5.14.3 +trino==0.334.0 +trove-classifiers==2025.5.9.12 +twine==6.1.0 +typer==0.15.3 +types-Deprecated==1.2.15.20250304 +types-Markdown==3.8.0.20250415 +types-PyMySQL==1.1.0.20241103 +types-PyYAML==6.0.12.20250402 +types-aiofiles==24.1.0.20250326 +types-certifi==2021.10.8.3 +types-cffi==1.17.0.20250326 +types-croniter==6.0.0.20250411 +types-docutils==0.21.0.20241128 +types-paramiko==3.5.0.20240928 +types-protobuf==6.30.2.20250506 +types-pyOpenSSL==24.1.0.20240722 +types-python-dateutil==2.9.0.20241206 +types-python-slugify==8.0.2.20240310 +types-pytz==2025.2.0.20250326 +types-redis==4.6.0.20241004 +types-requests==2.32.0.20250328 +types-setuptools==80.3.0.20250505 +types-tabulate==0.9.0.20241207 +types-toml==0.10.8.20240310 +typing-inspect==0.9.0 +typing-inspection==0.4.0 +typing_extensions==4.13.2 +tzdata==2025.2 +tzlocal==5.3.1 +uc-micro-py==1.0.3 +universal_pathlib==0.2.6 +uritemplate==4.1.1 +urllib3==2.4.0 +userpath==1.9.2 +uuid6==2024.7.10 +uv==0.7.3 +uvicorn==0.34.2 +uvloop==0.21.0 +validators==0.34.0 +vertica-python==1.4.0 +vine==5.1.0 +virtualenv==20.31.2 +watchfiles==1.0.5 +watchtower==3.4.0 +wcwidth==0.2.13 +weaviate-client==4.9.6 +webencodings==0.5.1 +websocket-client==1.8.0 +websockets==14.2 +wheel==0.45.1 +wirerope==1.0.0 +wrapt==1.17.2 +xmlsec==1.3.14 +xmltodict==0.14.2 +yamllint==1.37.1 +yandex-query-client==0.1.4 +yandexcloud==0.328.0 +yarl==1.20.0 +ydb-dbapi==0.1.11 +ydb==3.21.1 +zeep==4.3.1 +zenpy==2.0.56 +zipp==3.21.0 +zope.event==5.0 +zope.interface==7.2 +zstandard==0.23.0 diff --git a/airflow/versions.py b/airflow/versions.py index 21b34efb8..039bf19d6 100644 --- a/airflow/versions.py +++ b/airflow/versions.py @@ -15,4 +15,12 @@ "tini": "0.19.0", "vector": "0.43.1", }, + { + "product": "3.0.1", + "python": "3.12", + "git_sync": "v4.4.0", + "statsd_exporter": "0.28.0", + "tini": "0.19.0", + "vector": "0.43.1", + }, ] From 6b5d891335febb4034b945d34f50461629562262 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 20 May 2025 12:14:57 +0200 Subject: [PATCH 07/12] removed wip deadwood --- ...28242-Updates-async-profiler-support.patch | 25 +- ...rvername-with-external-host-and-port.patch | 519 ------------------ 2 files changed, 12 insertions(+), 532 deletions(-) delete mode 100644 hbase/stackable/patches/2.6.1/0005-extend-servername-with-external-host-and-port.patch diff --git a/hbase/stackable/patches/2.6.1/0001-HBASE-28242-Updates-async-profiler-support.patch b/hbase/stackable/patches/2.6.1/0001-HBASE-28242-Updates-async-profiler-support.patch index bf9d654ad..2901aaf99 100644 --- a/hbase/stackable/patches/2.6.1/0001-HBASE-28242-Updates-async-profiler-support.patch +++ b/hbase/stackable/patches/2.6.1/0001-HBASE-28242-Updates-async-profiler-support.patch @@ -4,8 +4,8 @@ Date: Tue, 6 Feb 2024 16:10:54 +0100 Subject: HBASE-28242: Updates async-profiler support --- - .../hadoop/hbase/http/ProfileServlet.java | 207 +++++++++++------- - 1 file changed, 122 insertions(+), 85 deletions(-) + .../hadoop/hbase/http/ProfileServlet.java | 205 +++++++++++------- + 1 file changed, 121 insertions(+), 84 deletions(-) diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileServlet.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileServlet.java index 83087dbed3..6139114717 100644 @@ -106,22 +106,21 @@ index 83087dbed3..6139114717 100644 ALLOC("alloc"), LOCK("lock"), - PAGE_FAULTS("page-faults"), -- CONTEXT_SWITCHES("context-switches"), -- CYCLES("cycles"), -- INSTRUCTIONS("instructions"), -- CACHE_REFERENCES("cache-references"), -- CACHE_MISSES("cache-misses"), -- BRANCHES("branches"), + ITIMER("itimer"), + BRANCH_INSTRUCTIONS("branch-instructions"), - BRANCH_MISSES("branch-misses"), - BUS_CYCLES("bus-cycles"), ++ BRANCH_MISSES("branch-misses"), ++ BUS_CYCLES("bus-cycles"), + CACHE_MISSES("cache-misses"), + CACHE_REFERENCES("cache-references"), -+ CONTEXT_SWITCHES("context-switches"), -+ CYCLES("cycles"), + CONTEXT_SWITCHES("context-switches"), + CYCLES("cycles"), + DTLB_LOAD_MISSES("dTLB-load-misses"), -+ INSTRUCTIONS("instructions"), + INSTRUCTIONS("instructions"), +- CACHE_REFERENCES("cache-references"), +- CACHE_MISSES("cache-misses"), +- BRANCHES("branches"), +- BRANCH_MISSES("branch-misses"), +- BUS_CYCLES("bus-cycles"), L1_DCACHE_LOAD_MISSES("L1-dcache-load-misses"), LLC_LOAD_MISSES("LLC-load-misses"), - DTLB_LOAD_MISSES("dTLB-load-misses"), diff --git a/hbase/stackable/patches/2.6.1/0005-extend-servername-with-external-host-and-port.patch b/hbase/stackable/patches/2.6.1/0005-extend-servername-with-external-host-and-port.patch deleted file mode 100644 index b8c2e09b9..000000000 --- a/hbase/stackable/patches/2.6.1/0005-extend-servername-with-external-host-and-port.patch +++ /dev/null @@ -1,519 +0,0 @@ -From 958fd8f17dd203329b1da058e4714762412b4c57 Mon Sep 17 00:00:00 2001 -From: Andrew Kenworthy -Date: Wed, 16 Apr 2025 16:43:33 +0200 -Subject: extend servername with external host and port - ---- - .../hbase/shaded/protobuf/ProtobufUtil.java | 15 ++++++- - .../org/apache/hadoop/hbase/ServerName.java | 43 ++++++++++++++++++- - .../src/main/protobuf/HBase.proto | 2 + - .../main/protobuf/RegionServerStatus.proto | 3 ++ - hbase-protocol/src/main/protobuf/HBase.proto | 2 + - .../src/main/protobuf/ZooKeeper.proto | 2 + - .../tmpl/master/RegionServerListTmpl.jamon | 2 +- - .../tmpl/regionserver/RSStatusTmpl.jamon | 2 +- - .../apache/hadoop/hbase/master/HMaster.java | 7 +++ - .../hadoop/hbase/master/ServerManager.java | 4 +- - .../hbase/regionserver/HRegionServer.java | 16 ++++++- - .../resources/hbase-webapps/master/hbck.jsp | 4 +- - .../hbase-webapps/master/rsgroup.jsp | 15 +++---- - .../resources/hbase-webapps/master/table.jsp | 30 ++++++------- - .../hbase/zookeeper/MasterAddressTracker.java | 3 ++ - 15 files changed, 116 insertions(+), 34 deletions(-) - -diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java -index 0fe181b503..be121ea641 100644 ---- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java -+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java -@@ -417,6 +417,13 @@ public final class ProtobufUtil { - if (serverName.getStartcode() >= 0) { - builder.setStartCode(serverName.getStartcode()); - } -+ -+ if (serverName.getExternalHostname() != null) { -+ builder.setExternalHostName(serverName.getExternalHostname()); -+ } -+ if (serverName.getExternalPort() != null) { -+ builder.setExternalPort(serverName.getExternalPort()); -+ } - return builder.build(); - } - -@@ -428,6 +435,9 @@ public final class ProtobufUtil { - public static ServerName toServerName(final HBaseProtos.ServerName proto) { - if (proto == null) return null; - String hostName = proto.getHostName(); -+ String externalHostName = proto.getExternalHostName(); -+ Integer externalPort = proto.getExternalPort(); -+ - long startCode = -1; - int port = -1; - if (proto.hasPort()) { -@@ -436,7 +446,7 @@ public final class ProtobufUtil { - if (proto.hasStartCode()) { - startCode = proto.getStartCode(); - } -- return ServerName.valueOf(hostName, port, startCode); -+ return ServerName.valueOf(hostName, port, startCode, externalHostName, externalPort); - } - - /** -@@ -3145,7 +3155,8 @@ public final class ProtobufUtil { - ZooKeeperProtos.Master.parser().parseFrom(data, prefixLen, data.length - prefixLen); - org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.ServerName sn = - rss.getMaster(); -- return ServerName.valueOf(sn.getHostName(), sn.getPort(), sn.getStartCode()); -+ return ServerName.valueOf(sn.getHostName(), sn.getPort(), sn.getStartCode(), -+ sn.getExternalHostName(), sn.getExternalPort()); - } catch (/* InvalidProtocolBufferException */IOException e) { - // A failed parse of the znode is pretty catastrophic. Rather than loop - // retrying hoping the bad bytes will changes, and rather than change -diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/ServerName.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/ServerName.java -index 5223bac3e5..979eb4c6ae 100644 ---- a/hbase-common/src/main/java/org/apache/hadoop/hbase/ServerName.java -+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ServerName.java -@@ -85,6 +85,8 @@ public class ServerName implements Comparable, Serializable { - private final String serverName; - private final long startCode; - private transient Address address; -+ private String externalHostname; -+ private Integer externalPort; - - /** - * Cached versioned bytes of this ServerName instance. -@@ -103,11 +105,21 @@ public class ServerName implements Comparable, Serializable { - this(Address.fromParts(hostname, port), startCode); - } - -- private ServerName(final Address address, final long startCode) { -+ protected ServerName(final String hostname, final int port, final long startCode, String externalHostname, Integer externalPort) { -+ this(Address.fromParts(hostname, port), startCode, externalHostname, externalPort); -+ } -+ -+ private ServerName(final Address address, final long startCode, String externalHostname, Integer externalPort) { - // Use HostAndPort to host port and hostname. Does validation and can do ipv6 - this.address = address; - this.startCode = startCode; - this.serverName = getServerName(this.address.getHostname(), this.address.getPort(), startCode); -+ this.externalHostname = externalHostname; -+ this.externalPort = externalPort; -+ } -+ -+ private ServerName(final Address address, final long startCode) { -+ this(address, startCode, null, null); - } - - private ServerName(final String hostAndPort, final long startCode) { -@@ -177,6 +189,11 @@ public class ServerName implements Comparable, Serializable { - return INTERN_POOL.intern(new ServerName(hostname, port, startCode)); - } - -+ public static ServerName valueOf(final String hostname, final int port, final long startCode, -+ String externalHostname, Integer externalPort) { -+ return new ServerName(hostname, port, startCode, externalHostname, externalPort); -+ } -+ - /** - * Retrieve an instance of ServerName. Callers should use the equals method to compare returned - * instances, though we may return a shared immutable object as an internal optimization. -@@ -253,6 +270,30 @@ public class ServerName implements Comparable, Serializable { - return this.address.getPort(); - } - -+ public String getExternalHostname() { -+ return this.externalHostname; -+ } -+ -+ public Integer getExternalPort() { -+ return this.externalPort; -+ } -+ -+ public String getDisplayHostname() { -+ return this.externalHostname != null && !this.externalHostname.isEmpty() ? this.externalHostname: this.getHostname(); -+ } -+ -+ public Integer getDisplayPort() { -+ return this.externalPort != null && this.externalPort > 0 ? this.externalPort: this.getPort(); -+ } -+ -+ public void setAddress(Address address) { -+ this.address = address; -+ } -+ -+ public void setExternalPort(Integer externalPort) { -+ this.externalPort = externalPort; -+ } -+ - /** - * Return the start code. - * @deprecated Since 2.5.0, will be removed in 4.0.0. Use {@link #getStartCode()} instead. -diff --git a/hbase-protocol-shaded/src/main/protobuf/HBase.proto b/hbase-protocol-shaded/src/main/protobuf/HBase.proto -index e858d7ec33..2da62e0e0a 100644 ---- a/hbase-protocol-shaded/src/main/protobuf/HBase.proto -+++ b/hbase-protocol-shaded/src/main/protobuf/HBase.proto -@@ -147,6 +147,8 @@ message ServerName { - required string host_name = 1; - optional uint32 port = 2; - optional uint64 start_code = 3; -+ optional string external_host_name = 100; -+ optional uint32 external_port = 101; - } - - // Comment data structures -diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto -index 53751082f9..ecb3cf6221 100644 ---- a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto -+++ b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto -@@ -42,6 +42,9 @@ message RegionServerStartupRequest { - - /** hostname for region server, optional */ - optional string use_this_hostname_instead = 4; -+ -+ optional string external_host_name = 100; -+ optional uint32 external_port = 101; - } - - message RegionServerStartupResponse { -diff --git a/hbase-protocol/src/main/protobuf/HBase.proto b/hbase-protocol/src/main/protobuf/HBase.proto -index ecfdc8c2c7..833cee7bee 100644 ---- a/hbase-protocol/src/main/protobuf/HBase.proto -+++ b/hbase-protocol/src/main/protobuf/HBase.proto -@@ -135,6 +135,8 @@ message ServerName { - required string host_name = 1; - optional uint32 port = 2; - optional uint64 start_code = 3; -+ optional string external_host_name = 100; -+ optional uint32 external_port = 101; - } - - // Comment data structures -diff --git a/hbase-protocol/src/main/protobuf/ZooKeeper.proto b/hbase-protocol/src/main/protobuf/ZooKeeper.proto -index c0f40e970e..bddadf4d30 100644 ---- a/hbase-protocol/src/main/protobuf/ZooKeeper.proto -+++ b/hbase-protocol/src/main/protobuf/ZooKeeper.proto -@@ -56,6 +56,8 @@ message Master { - // Major RPC version so that clients can know what version the master can accept. - optional uint32 rpc_version = 2; - optional uint32 info_port = 3; -+ optional string external_host = 4; -+ optional uint32 external_port = 5; - } - - /** -diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon -index 4c9140470b..641f558d87 100644 ---- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon -+++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon -@@ -500,7 +500,7 @@ if (totalCompactingCells > 0) { - - <%java> - int infoPort = master.getRegionServerInfoPort(serverName); -- String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; -+ String url = "//" + serverName.getDisplayHostname() + ":" + serverName.getDisplayPort() + "/rs-status"; - - - <%if infoPort > 0%> -diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon -index 23330096cf..3abe5d617d 100644 ---- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon -+++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon -@@ -245,7 +245,7 @@ org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; - No master found - <%else> - <%java> -- String host = masterServerName.getHostname() + ":" + infoPort; -+ String host = masterServerName.getDisplayHostname() + ":" + masterServerName.getDisplayPort(); - String url = "//" + host + "/master-status"; - - <% host %> -diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java -index 3fe5abac27..652c709ab5 100644 ---- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java -+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java -@@ -3128,6 +3128,13 @@ public class HMaster extends HRegionServer implements MasterServices { - - public int getRegionServerInfoPort(final ServerName sn) { - int port = this.serverManager.getInfoPort(sn); -+ if (port == 0) { -+ if (sn.getExternalPort() != null) { -+ port = sn.getExternalPort(); -+ } else { -+ port = sn.getAddress().getPort(); -+ } -+ } - return port == 0 - ? conf.getInt(HConstants.REGIONSERVER_INFO_PORT, HConstants.DEFAULT_REGIONSERVER_INFOPORT) - : port; -diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java -index f7115a5cef..e5f6062793 100644 ---- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java -+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java -@@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.ServerName; - import org.apache.hadoop.hbase.YouAreDeadException; - import org.apache.hadoop.hbase.client.ClusterConnection; - import org.apache.hadoop.hbase.client.RegionInfo; -+import org.apache.hadoop.hbase.client.RetriesExhaustedException; - import org.apache.hadoop.hbase.conf.ConfigurationObserver; - import org.apache.hadoop.hbase.ipc.DecommissionedHostRejectedException; - import org.apache.hadoop.hbase.ipc.HBaseRpcController; -@@ -230,7 +231,8 @@ public class ServerManager implements ConfigurationObserver { - String isaHostName = useIp ? ia.getHostAddress() : ia.getHostName(); - final String hostname = - request.hasUseThisHostnameInstead() ? request.getUseThisHostnameInstead() : isaHostName; -- ServerName sn = ServerName.valueOf(hostname, request.getPort(), request.getServerStartCode()); -+ ServerName sn = ServerName.valueOf(hostname, request.getPort(), request.getServerStartCode(), -+ request.getExternalHostName(), request.getExternalPort()); - - // Check if the host should be rejected based on it's decommissioned status - checkRejectableDecommissionedStatus(sn); -diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java -index 27bcef2f06..8438378059 100644 ---- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java -+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java -@@ -678,7 +678,18 @@ public class HRegionServer extends Thread - useIp ? rpcServices.isa.getAddress().getHostAddress() : rpcServices.isa.getHostName(); - String hostName = - StringUtils.isBlank(useThisHostnameInstead) ? isaHostName : useThisHostnameInstead; -- serverName = ServerName.valueOf(hostName, this.rpcServices.isa.getPort(), this.startcode); -+ -+ String externalHostname; -+ Integer externalPort; -+ if (this instanceof HMaster) { -+ externalHostname = conf.get("hbase.listener.master.hostname", ""); -+ externalPort = conf.getInt("hbase.listener.master.port", -1); -+ } else { -+ externalHostname = conf.get("hbase.listener.regionserver.hostname", ""); -+ externalPort = conf.getInt("hbase.listener.regionserver.port", -1); -+ } -+ -+ serverName = ServerName.valueOf(hostName, this.rpcServices.isa.getPort(), this.startcode, externalHostname, externalPort); - - rpcControllerFactory = RpcControllerFactory.instantiate(this.conf); - rpcRetryingCallerFactory = RpcRetryingCallerFactory.instantiate(this.conf, -@@ -3081,6 +3092,9 @@ public class HRegionServer extends Thread - request.setPort(port); - request.setServerStartCode(this.startcode); - request.setServerCurrentTime(now); -+ request.setExternalHostName(serverName.getExternalHostname()); -+ request.setExternalPort(serverName.getExternalPort()); -+ - result = rss.regionServerStartup(null, request.build()); - } catch (ServiceException se) { - IOException ioe = ProtobufUtil.getRemoteException(se); -diff --git a/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp b/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp -index 38e16ca8e2..61e4cf2b17 100644 ---- a/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp -+++ b/hbase-server/src/main/resources/hbase-webapps/master/hbck.jsp -@@ -364,9 +364,9 @@ private static String formatServerName(HMaster master, - } - String sn = serverName.toString(); - if (serverManager.isServerOnline(serverName)) { -- int infoPort = master.getRegionServerInfoPort(serverName); -+ int infoPort = serverName.getDisplayPort(); - if (infoPort > 0) { -- return "" + sn + ""; - } else { - return "" + sn + ""; -diff --git a/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp b/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp -index 6ba6f78a27..d05e1bc8f0 100644 ---- a/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp -+++ b/hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp -@@ -170,8 +170,7 @@ - lastContact = (System.currentTimeMillis() - sl.getReportTimestamp())/1000; - } - long startcode = serverName.getStartcode(); -- int infoPort = master.getRegionServerInfoPort(serverName); -- String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status";%> -+ String url = "//" + serverName.getDisplayHostname() + ":" + serverName.getDisplayPort() + "/rs-status";%> - - <%= serverName.getServerName() %> - <%= new Date(startcode) %> -@@ -225,8 +224,7 @@ - double memStoreSizeMB = sl.getRegionMetrics().values() - .stream().mapToDouble(rm -> rm.getMemStoreSize().get(Size.Unit.MEGABYTE)) - .sum(); -- int infoPort = master.getRegionServerInfoPort(serverName); -- String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; -+ String url = "//" + serverName.getDisplayHostname() + ":" + serverName.getDisplayPort() + "/rs-status"; - - if (memStoreSizeMB > 0) { - memStoreSizeMBStr = TraditionalBinaryPrefix.long2String( -@@ -270,14 +268,13 @@ - ServerName serverName = serverMaping.get(server); - ServerMetrics sl = onlineServers.get(server); - if (sl != null && serverName != null) { -- int infoPort = master.getRegionServerInfoPort(serverName); - long readRequestCount = 0; - long writeRequestCount = 0; - for (RegionMetrics rm : sl.getRegionMetrics().values()) { - readRequestCount += rm.getReadRequestCount(); - writeRequestCount += rm.getWriteRequestCount(); - } -- String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; -+ String url = "//" + serverName.getDisplayHostname() + ":" + serverName.getDisplayPort() + "/rs-status"; - %> - - <%= serverName.getServerName() %> -@@ -330,8 +327,7 @@ - totalStaticIndexSizeKB += rm.getStoreFileUncompressedDataIndexSize().get(Size.Unit.KILOBYTE); - totalStaticBloomSizeKB += rm.getBloomFilterSize().get(Size.Unit.KILOBYTE); - } -- int infoPort = master.getRegionServerInfoPort(serverName); -- String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; -+ String url = "//" + serverName.getDisplayHostname() + ":" + serverName.getDisplayPort() + "/rs-status"; - if (storeUncompressedSizeMB > 0) { - storeUncompressedSizeMBStr = TraditionalBinaryPrefix.long2String( - (long) storeUncompressedSizeMB * TraditionalBinaryPrefix.MEGA.value, "B", 1); -@@ -396,8 +392,7 @@ - percentDone = String.format("%.2f", 100 * - ((float) currentCompactedCells / totalCompactingCells)) + "%"; - } -- int infoPort = master.getRegionServerInfoPort(serverName); -- String url = "//" + serverName.getHostname() + ":" + infoPort + "/rs-status"; -+ String url = "//" + serverName.getDisplayHostname() + ":" + serverName.getDisplayPort() + "/rs-status"; - %> - - <%= serverName.getServerName() %> -diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp -index 56a0c8096d..01fbde1b18 100644 ---- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp -+++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp -@@ -93,8 +93,8 @@ - * its region server. - * @return an anchor tag if one can be built, {@code null} otherwise. - */ -- private static String buildRegionServerLink(final ServerName serverName, final int rsInfoPort, -- final RegionInfo regionInfo, final RegionState.State regionState) { -+ private static String buildRegionServerLink(final ServerName serverName, -+ final RegionInfo regionInfo, final RegionState.State regionState) { - if (serverName == null || regionInfo == null) { return null; } - - if (regionState != RegionState.State.OPEN) { -@@ -102,7 +102,7 @@ - return serverName.getServerName(); - } - -- final String socketAddress = serverName.getHostname() + ":" + rsInfoPort; -+ final String socketAddress = serverName.getDisplayHostname() + ":" + serverName.getDisplayPort(); - final String URI = "//" + socketAddress + "/region.jsp" - + "?name=" + regionInfo.getEncodedName(); - return "" + serverName.getServerName() + ""; -@@ -299,7 +299,7 @@ - if (metaLocation != null) { - ServerMetrics sl = master.getServerManager().getLoad(metaLocation); - // The host name portion should be safe, but I don't know how we handle IDNs so err on the side of failing safely. -- hostAndPort = URLEncoder.encode(metaLocation.getHostname()) + ":" + master.getRegionServerInfoPort(metaLocation); -+ hostAndPort = URLEncoder.encode(metaLocation.getDisplayHostname()) + ":" + metaLocation.getDisplayPort().toString(); - if (sl != null) { - Map map = sl.getRegionMetrics(); - if (map.containsKey(meta.getRegionName())) { -@@ -374,7 +374,7 @@ - float localityForSsd = 0.0f; - if (metaLocation != null) { - ServerMetrics sl = master.getServerManager().getLoad(metaLocation); -- hostAndPort = URLEncoder.encode(metaLocation.getHostname()) + ":" + master.getRegionServerInfoPort(metaLocation); -+ hostAndPort = URLEncoder.encode(metaLocation.getDisplayHostname()) + ":" + metaLocation.getDisplayPort().toString(); - if (sl != null) { - Map map = sl.getRegionMetrics(); - if (map.containsKey(meta.getRegionName())) { -@@ -426,7 +426,7 @@ - String compactionProgress = ""; - if (metaLocation != null) { - ServerMetrics sl = master.getServerManager().getLoad(metaLocation); -- hostAndPort = URLEncoder.encode(metaLocation.getHostname()) + ":" + master.getRegionServerInfoPort(metaLocation); -+ hostAndPort = URLEncoder.encode(metaLocation.getDisplayHostname()) + ":" + metaLocation.getDisplayPort().toString(); - if (sl != null) { - Map map = sl.getRegionMetrics(); - if (map.containsKey(meta.getRegionName())) { -@@ -548,7 +548,7 @@ - <%= endKeyDisplay %> - <%= replicaIdDisplay %> - <%= regionStateDisplay %> -- "><%= serverName != null ? buildRegionServerLink(serverName, master.getRegionServerInfoPort(serverName), regionInfo, regionState) : "" %> -+ "><%= serverName != null ? buildRegionServerLink(serverName, regionInfo, regionState) : "" %> - <%= seqNum %> - <%= targetServerName %> - <%= mergeRegionNames %> -@@ -951,7 +951,7 @@ - ServerMetrics sl = master.getServerManager().getLoad(addr); - // This port might be wrong if RS actually ended up using something else. - urlRegionServer = -- "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; -+ "//" + URLEncoder.encode(addr.getDisplayHostname()) + ":" + addr.getDisplayPort().toString() + "/rs-status"; - if(sl != null) { - Integer i = regDistribution.get(addr); - if (null == i) i = Integer.valueOf(0); -@@ -972,7 +972,7 @@ - if (urlRegionServer != null) { - %> - -- <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> -+ <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getDisplayHostname().toString()) + ":" + addr.getDisplayPort().toString() %> - - <% - } else { -@@ -1037,7 +1037,7 @@ - if (addr != null) { - // This port might be wrong if RS actually ended up using something else. - urlRegionServer = -- "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; -+ "//" + URLEncoder.encode(addr.getDisplayHostname()) + ":" + addr.getDisplayPort().toString() + "/rs-status"; - } - if (numRegionsRendered < numRegionsToRender) { - numRegionsRendered++; -@@ -1048,7 +1048,7 @@ - if (urlRegionServer != null) { - %> - -- <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> -+ <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getDisplayHostname().toString()) + ":" + addr.getDisplayPort().toString() %> - - <% - } else { -@@ -1098,7 +1098,7 @@ - if (addr != null) { - // This port might be wrong if RS actually ended up using something else. - urlRegionServer = -- "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; -+ "//" + URLEncoder.encode(addr.getDisplayHostname()) + ":" + addr.getDisplayPort().toString() + "/rs-status"; - } - if (numRegionsRendered < numRegionsToRender) { - numRegionsRendered++; -@@ -1109,7 +1109,7 @@ - if (urlRegionServer != null) { - %> - -- <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> -+ <%= addr == null? "-": StringEscapeUtils.escapeHtml4(addr.getDisplayHostname().toString()) + ":" + addr.getDisplayPort().toString() %> - - <% - } else { -@@ -1153,10 +1153,10 @@ - <% - for (Map.Entry rdEntry : regDistribution.entrySet()) { - ServerName addr = rdEntry.getKey(); -- String url = "//" + URLEncoder.encode(addr.getHostname()) + ":" + master.getRegionServerInfoPort(addr) + "/rs-status"; -+ String url = "//" + URLEncoder.encode(addr.getDisplayHostname()) + ":" + addr.getDisplayPort().toString() + "/rs-status"; - %> - -- <%= StringEscapeUtils.escapeHtml4(addr.getHostname().toString()) + ":" + master.getRegionServerInfoPort(addr) %> -+ <%= StringEscapeUtils.escapeHtml4(addr.getDisplayHostname().toString()) + ":" + addr.getDisplayPort().toString() %> - <%= rdEntry.getValue()%> - <% - if (withReplica) { -diff --git a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterAddressTracker.java b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterAddressTracker.java -index 840ee2d215..b0f9da3e00 100644 ---- a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterAddressTracker.java -+++ b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MasterAddressTracker.java -@@ -281,6 +281,9 @@ public class MasterAddressTracker extends ZKNodeTracker { - snbuilder.setHostName(sn.getHostname()); - snbuilder.setPort(sn.getPort()); - snbuilder.setStartCode(sn.getStartcode()); -+ snbuilder.setExternalHostName(sn.getExternalHostname()); -+ snbuilder.setExternalPort(sn.getExternalPort()); -+ - mbuilder.setMaster(snbuilder.build()); - mbuilder.setRpcVersion(HConstants.RPC_CURRENT_VERSION); - mbuilder.setInfoPort(infoPort); From 1907ee0bb343f7600067a6c92abb7b5bb85b2983 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 20 May 2025 12:15:09 +0200 Subject: [PATCH 08/12] removed wip deadwood II --- hbase/stackable/bin/hbase-entrypoint.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/hbase/stackable/bin/hbase-entrypoint.sh b/hbase/stackable/bin/hbase-entrypoint.sh index 14d96bb61..3b5192016 100755 --- a/hbase/stackable/bin/hbase-entrypoint.sh +++ b/hbase/stackable/bin/hbase-entrypoint.sh @@ -16,8 +16,6 @@ HBASE_ROLE_NAME="$1" HBASE_ROLE_SERVICE_NAME="$2" # 16010 for master, 16020 for regionservers etc. HBASE_ROLE_SERVICE_PORT="$3" -# ui-http or ui-https -HBASE_PORT_NAME="$4" HBASE_ROLE_SERVICE_HOST="${HOSTNAME}.${HBASE_ROLE_SERVICE_NAME}" @@ -66,7 +64,6 @@ cp /stackable/tmp/hdfs/core-site.xml /stackable/conf cp /stackable/tmp/hbase/* /stackable/conf cp /stackable/tmp/log_config/log4j* /stackable/conf -# Kerberos if [ -f /stackable/kerberos/krb5.conf ]; then KERBEROS_REALM=$(grep -oP 'default_realm = \K.*' /stackable/kerberos/krb5.conf) export KERBEROS_REALM @@ -75,12 +72,6 @@ if [ -f /stackable/kerberos/krb5.conf ]; then sed -i -e s/\$\{env\.KERBEROS_REALM\}/"${KERBEROS_REALM}"/g /stackable/conf/hdfs-site.xml fi -# Service endpoints -HBASE_SERVICE_HOST=$(cat /stackable/listener/default-address/address) -HBASE_SERVICE_PORT=$(cat /stackable/listener/default-address/ports/"${HBASE_PORT_NAME}") -sed -i -e s/\$\{HBASE_SERVICE_HOST\}/"${HBASE_SERVICE_HOST}"/g /stackable/conf/hbase-site.xml -sed -i -e s/\$\{HBASE_SERVICE_PORT\}/"${HBASE_SERVICE_PORT}"/g /stackable/conf/hbase-site.xml - rm -f "${STACKABLE_LOG_DIR}/_vector/shutdown" prepare_signal_handlers /stackable/containerdebug --output="${STACKABLE_LOG_DIR}/containerdebug-state.json" --loop & From d451599aa2633e72c7302c608a2abb8000e3c90f Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 20 May 2025 15:36:16 +0200 Subject: [PATCH 09/12] move extras to argument --- airflow/Dockerfile | 8 +++++++- airflow/versions.py | 11 +++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/airflow/Dockerfile b/airflow/Dockerfile index 3c2c189b6..9d862068c 100644 --- a/airflow/Dockerfile +++ b/airflow/Dockerfile @@ -32,10 +32,16 @@ ARG STACKABLE_USER_UID ARG S3FS ARG CYCLONEDX_BOM ARG UV +ARG EXTRAS +COPY airflow/constraints-${PRODUCT}-python${PYTHON}.txt /tmp/constraints.txt +COPY --from=opa-auth-manager-builder /tmp/opa-auth-manager/dist/opa_auth_manager-0.1.0-py3-none-any.whl /tmp/ + +# Airflow "extras" packages are listed here: https://airflow.apache.org/docs/apache-airflow/stable/extra-packages-ref.html +# They evolve over time and thus belong to the version-specific arguments. # The mysql provider is currently excluded. # Requires implementation of https://github.com/apache/airflow/blob/2.2.5/scripts/docker/install_mysql.sh -ENV AIRFLOW_EXTRAS=async,amazon,celery,cncf-kubernetes,docker,elasticsearch,fab,ftp,grpc,hashicorp,http,ldap,google,microsoft-azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,trino +ENV AIRFLOW_EXTRAS=${EXTRAS} RUN microdnf update && \ microdnf install \ diff --git a/airflow/versions.py b/airflow/versions.py index 9f536ad86..88f3f3c51 100644 --- a/airflow/versions.py +++ b/airflow/versions.py @@ -9,6 +9,7 @@ "tini": "0.19.0", "vector": "0.46.1", "uv": "0.7.3", + "extras": "async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,ldap,google,google_auth,microsoft.azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,trino", }, { "product": "2.10.4", @@ -20,6 +21,7 @@ "tini": "0.19.0", "vector": "0.46.1", "uv": "0.7.3", + "extras": "async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,ldap,google,google_auth,microsoft.azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,trino", }, { "product": "2.10.5", @@ -31,13 +33,18 @@ "tini": "0.19.0", "vector": "0.46.1", "uv": "0.7.3", + "extras": "async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,ldap,google,google_auth,microsoft.azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,trino", }, { "product": "3.0.1", "python": "3.12", "git_sync": "v4.4.0", - "statsd_exporter": "0.28.0", + "s3fs": "2024.9.0", + "cyclonedx_bom": "6.0.0", + "shared/statsd-exporter": "0.28.0", "tini": "0.19.0", - "vector": "0.43.1", + "vector": "0.46.1", + "uv": "0.7.3", + "extras": "async,amazon,celery,cncf-kubernetes,docker,elasticsearch,fab,ftp,grpc,hashicorp,http,ldap,google,microsoft-azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,trino", }, ] From 8903d97b3c42d81701aa519ef5f9c1db06be7767 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 20 May 2025 16:31:33 +0200 Subject: [PATCH 10/12] re-org constraints --- airflow/Dockerfile | 2 +- .../constraints/3.0.1/build-constraints-python3.12.txt | 1 + .../constraints/3.0.1/constraints-python3.12.txt} | 0 3 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 airflow/stackable/constraints/3.0.1/build-constraints-python3.12.txt rename airflow/{constraints-3.0.1-python3.12.txt => stackable/constraints/3.0.1/constraints-python3.12.txt} (100%) diff --git a/airflow/Dockerfile b/airflow/Dockerfile index 9d862068c..59626974f 100644 --- a/airflow/Dockerfile +++ b/airflow/Dockerfile @@ -34,7 +34,7 @@ ARG CYCLONEDX_BOM ARG UV ARG EXTRAS -COPY airflow/constraints-${PRODUCT}-python${PYTHON}.txt /tmp/constraints.txt +COPY airflow/stackable/constraints/${PRODUCT}/constraints-python${PYTHON}.txt /tmp/constraints.txt COPY --from=opa-auth-manager-builder /tmp/opa-auth-manager/dist/opa_auth_manager-0.1.0-py3-none-any.whl /tmp/ # Airflow "extras" packages are listed here: https://airflow.apache.org/docs/apache-airflow/stable/extra-packages-ref.html diff --git a/airflow/stackable/constraints/3.0.1/build-constraints-python3.12.txt b/airflow/stackable/constraints/3.0.1/build-constraints-python3.12.txt new file mode 100644 index 000000000..0d2d86dd6 --- /dev/null +++ b/airflow/stackable/constraints/3.0.1/build-constraints-python3.12.txt @@ -0,0 +1 @@ +Cython==3.1.0 diff --git a/airflow/constraints-3.0.1-python3.12.txt b/airflow/stackable/constraints/3.0.1/constraints-python3.12.txt similarity index 100% rename from airflow/constraints-3.0.1-python3.12.txt rename to airflow/stackable/constraints/3.0.1/constraints-python3.12.txt From d2c647cff06c0cf20794c24fb5216158af1a4412 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 20 May 2025 17:19:51 +0200 Subject: [PATCH 11/12] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a11588c3..a47ab1860 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ All notable changes to this project will be documented in this file. - airflow: check for correct permissions and ownerships in /stackable folder via `check-permissions-ownership.sh` provided in stackable-base image ([#1054]). - airflow: Add `2.10.5` ([#1108]). +- airflow: Add `3.0.1` ([#1122]). - druid: check for correct permissions and ownerships in /stackable folder via `check-permissions-ownership.sh` provided in stackable-base image ([#1039]). - druid: Add `33.0.0` ([#1110]). @@ -135,6 +136,7 @@ All notable changes to this project will be documented in this file. [#1116]: https://github.com/stackabletech/docker-images/pull/1116 [#1119]: https://github.com/stackabletech/docker-images/pull/1119 [#1121]: https://github.com/stackabletech/docker-images/pull/1121 +[#1122]: https://github.com/stackabletech/docker-images/pull/1122 ## [25.3.0] - 2025-03-21 From 8ffd0e3b8b6119df107ce3fb1fe140f2cdfdd953 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 23 May 2025 15:49:08 +0200 Subject: [PATCH 12/12] removed duplicate lines --- airflow/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/airflow/Dockerfile b/airflow/Dockerfile index 675ff2309..bd8c48ef8 100644 --- a/airflow/Dockerfile +++ b/airflow/Dockerfile @@ -34,9 +34,6 @@ ARG CYCLONEDX_BOM ARG UV ARG EXTRAS -COPY airflow/stackable/constraints/${PRODUCT}/constraints-python${PYTHON}.txt /tmp/constraints.txt -COPY --from=opa-auth-manager-builder /tmp/opa-auth-manager/dist/opa_auth_manager-0.1.0-py3-none-any.whl /tmp/ - # Airflow "extras" packages are listed here: https://airflow.apache.org/docs/apache-airflow/stable/extra-packages-ref.html # They evolve over time and thus belong to the version-specific arguments. # The mysql provider is currently excluded.