RANGER-4898: docker setup updated to use kerberos authentication

.github/workflows/ci.yml

@@ -153,7 +153,7 @@ jobs:
       - name: Bring up containers
         run: |
           cd dev-support/ranger-docker
-          ./scripts/ozone-plugin-docker-setup.sh
+          ./scripts/ozone/ozone-plugin-docker-setup.sh
           export RANGER_DB_TYPE=postgres
           docker compose \
           -f docker-compose.ranger.yml \
@@ -177,7 +177,9 @@ jobs:
                   echo "Container $container is running!";
               else
                   flag=false;
-                  echo "Container $container is NOT running!";
+                  echo "Container $container is NOT running! Status: $(docker inspect -f '{{.State.Status}}' $container 2>/dev/null)";
+                  echo "docker logs $container:";
+                  docker logs $container
               fi
           done
 

dev-support/ranger-docker/.env

@@ -8,9 +8,9 @@ BUILD_OPTS=
 # DockerHub Ranger Base Image
 RANGER_BASE_IMAGE=apache/ranger-base
 # Java version used to run Ranger and dependent services is present as suffix: -8, valid values for suffix: -8, -11, -17
-RANGER_BASE_VERSION=20250707-1-8
+RANGER_BASE_VERSION=20251023-1-8
 # Java version used to build Apache Ranger is present as suffix: -8, valid values for suffix: -8, -11, -17
-RANGER_BASE_BUILD_VERSION=20250707-1-8
+RANGER_BASE_BUILD_VERSION=20251023-1-8
 RANGER_VERSION=3.0.0-SNAPSHOT
 
 # Hadoop Configuration
@@ -53,6 +53,14 @@ SOLR_VERSION=8.11.2
 # Zookeeper Configuration
 ZK_VERSION=3.8.4
 
+# Kerberos
+KERBEROS_ENABLED=true
+KERBEROS_REALM=EXAMPLE.COM
+KERBEROS_KDC_HOST=ranger-kdc.example.com
+KERBEROS_MASTER_PASSWORD=rangerR0cks!
+KERBEROS_ADMIN_PRINCIPAL=admin/admin
+KERBEROS_ADMIN_PASSWORD=rangerR0cks!
+
 # Database Versions
 POSTGRES_VERSION=13.16
 MARIADB_VERSION=10.6

dev-support/ranger-docker/Dockerfile.ranger

@@ -23,12 +23,17 @@ ARG RANGER_VERSION
 ARG RANGER_DB_TYPE
 ARG TARGETARCH
 
+VOLUME /etc/keytabs
+
 COPY ./dist/version                               /home/ranger/dist/
 COPY ./dist/ranger-${RANGER_VERSION}-admin.tar.gz /home/ranger/dist/
 
-COPY ./scripts/ranger.sh                                         ${RANGER_SCRIPTS}/
-COPY ./scripts/ranger-admin-install-${RANGER_DB_TYPE}.properties ${RANGER_SCRIPTS}/ranger-admin-install.properties
-COPY ./scripts/create-ranger-services.py                         ${RANGER_SCRIPTS}/
+COPY ./scripts/admin/ranger.sh                                         ${RANGER_SCRIPTS}/
+COPY ./scripts/admin/ranger-admin-install-${RANGER_DB_TYPE}.properties ${RANGER_SCRIPTS}/ranger-admin-install.properties
+COPY ./scripts/admin/create-ranger-services.py                         ${RANGER_SCRIPTS}/
+COPY ./scripts/hadoop/core-site.xml                                    ${RANGER_SCRIPTS}/
+COPY ./scripts/wait_for_keytab.sh                                      ${RANGER_SCRIPTS}/
+COPY ./scripts/kdc/krb5.conf                                           /etc/krb5.conf
 
 RUN    tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-admin.tar.gz --directory=${RANGER_HOME} \
     && ln -s ${RANGER_HOME}/ranger-${RANGER_VERSION}-admin ${RANGER_HOME}/admin \
@@ -37,7 +42,7 @@ RUN    tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-admin.tar.gz --direct
     && mkdir -p /var/run/ranger \
     && mkdir -p /var/log/ranger \
     && chown -R ranger:ranger ${RANGER_HOME}/admin/ ${RANGER_SCRIPTS}/ /var/run/ranger/ /var/log/ranger/ \
-    && chmod 755 ${RANGER_SCRIPTS}/ranger.sh \
+    && chmod 755 ${RANGER_SCRIPTS}/ranger.sh ${RANGER_SCRIPTS}/wait_for_keytab.sh \
     && mkdir -p /usr/share/java/
 
 FROM ranger AS ranger_postgres

dev-support/ranger-docker/Dockerfile.ranger-build

@@ -37,10 +37,10 @@ ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/
 RUN mkdir -p /home/ranger/git /home/ranger/.m2 && \
 	chown -R ranger:ranger /home/ranger
 
-COPY ./scripts/ranger-build.sh /home/ranger/scripts/
+COPY ./scripts/build/* ${RANGER_SCRIPTS}/
+RUN chmod 755 ${RANGER_SCRIPTS}/ranger-build.sh
 
 VOLUME /home/ranger/.m2
-VOLUME /home/ranger/scripts
 VOLUME /home/ranger/patches
 VOLUME /home/ranger/dist
 VOLUME /home/ranger/src

dev-support/ranger-docker/Dockerfile.ranger-hadoop

@@ -23,18 +23,25 @@ ARG HDFS_PLUGIN_VERSION
 ARG YARN_PLUGIN_VERSION
 ARG TEZ_VERSION
 
+VOLUME /etc/keytabs
 
 COPY ./dist/version                                          /home/ranger/dist/
 COPY ./dist/ranger-${HDFS_PLUGIN_VERSION}-hdfs-plugin.tar.gz /home/ranger/dist/
 COPY ./dist/ranger-${YARN_PLUGIN_VERSION}-yarn-plugin.tar.gz /home/ranger/dist/
 COPY ./downloads/hadoop-${HADOOP_VERSION}.tar.gz             /home/ranger/dist/
 COPY ./downloads/apache-tez-${TEZ_VERSION}-bin.tar.gz        /home/ranger/dist/
 
-COPY ./scripts/ranger-hadoop-setup.sh                   /home/ranger/scripts/
-COPY ./scripts/ranger-hadoop.sh                         /home/ranger/scripts/
-COPY ./scripts/ranger-hadoop-mkdir.sh                   /home/ranger/scripts/
-COPY ./scripts/ranger-hdfs-plugin-install.properties    /home/ranger/scripts/
-COPY ./scripts/ranger-yarn-plugin-install.properties    /home/ranger/scripts/
+COPY ./scripts/hadoop/ranger-hadoop-setup.sh                ${RANGER_SCRIPTS}/
+COPY ./scripts/hadoop/ranger-hadoop.sh                      ${RANGER_SCRIPTS}/
+COPY ./scripts/hadoop/ranger-hadoop-mkdir.sh                ${RANGER_SCRIPTS}/
+COPY ./scripts/hadoop/ranger-hadoop-healthcheck.sh          ${RANGER_SCRIPTS}/
+COPY ./scripts/hadoop/ranger-hdfs-plugin-install.properties ${RANGER_SCRIPTS}/
+COPY ./scripts/hadoop/ranger-yarn-plugin-install.properties ${RANGER_SCRIPTS}/
+COPY ./scripts/hadoop/core-site.xml                         ${RANGER_SCRIPTS}/
+COPY ./scripts/hadoop/hdfs-site.xml                         ${RANGER_SCRIPTS}/
+COPY ./scripts/hadoop/yarn-site.xml                         ${RANGER_SCRIPTS}/
+COPY ./scripts/wait_for_keytab.sh                           ${RANGER_SCRIPTS}/
+COPY ./scripts/kdc/krb5.conf                                /etc/krb5.conf
 
 RUN tar xvfz /home/ranger/dist/hadoop-${HADOOP_VERSION}.tar.gz --directory=/opt/ && \
     ln -s /opt/hadoop-${HADOOP_VERSION} /opt/hadoop && \
@@ -45,12 +52,16 @@ RUN tar xvfz /home/ranger/dist/hadoop-${HADOOP_VERSION}.tar.gz --directory=/opt/
     tar xvfz /home/ranger/dist/ranger-${HDFS_PLUGIN_VERSION}-hdfs-plugin.tar.gz --directory=/opt/ranger && \
     ln -s /opt/ranger/ranger-${HDFS_PLUGIN_VERSION}-hdfs-plugin /opt/ranger/ranger-hdfs-plugin && \
     rm -f /home/ranger/dist/ranger-${HDFS_PLUGIN_VERSION}-hdfs-plugin.tar.gz && \
-    cp -f /home/ranger/scripts/ranger-hdfs-plugin-install.properties /opt/ranger/ranger-hdfs-plugin/install.properties && \
+    cp -f ${RANGER_SCRIPTS}/ranger-hdfs-plugin-install.properties /opt/ranger/ranger-hdfs-plugin/install.properties && \
     tar xvfz /home/ranger/dist/ranger-${YARN_PLUGIN_VERSION}-yarn-plugin.tar.gz --directory=/opt/ranger && \
     ln -s /opt/ranger/ranger-${YARN_PLUGIN_VERSION}-yarn-plugin /opt/ranger/ranger-yarn-plugin && \
     rm -f /home/ranger/dist/ranger-${YARN_PLUGIN_VERSION}-yarn-plugin.tar.gz && \
-    cp -f /home/ranger/scripts/ranger-yarn-plugin-install.properties /opt/ranger/ranger-yarn-plugin/install.properties && \
+    cp -f ${RANGER_SCRIPTS}/ranger-yarn-plugin-install.properties /opt/ranger/ranger-yarn-plugin/install.properties && \
     chmod 744 ${RANGER_SCRIPTS}/ranger-hadoop-setup.sh ${RANGER_SCRIPTS}/ranger-hadoop.sh ${RANGER_SCRIPTS}/ranger-hadoop-mkdir.sh && \
+    useradd -g hadoop -ms /bin/bash healthcheck && \
+    chmod 744 ${RANGER_SCRIPTS}/ranger-hadoop-healthcheck.sh && \
+    chmod 755 ${RANGER_SCRIPTS}/wait_for_keytab.sh && \
+    chown healthcheck:hadoop ${RANGER_SCRIPTS}/ranger-hadoop-healthcheck.sh && \
     chown hdfs:hadoop ${RANGER_SCRIPTS}/ranger-hadoop-mkdir.sh
 
 RUN apt-get update && \

dev-support/ranger-docker/Dockerfile.ranger-hbase

@@ -21,23 +21,28 @@ FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION}
 ARG HBASE_VERSION
 ARG HBASE_PLUGIN_VERSION
 
+VOLUME /etc/keytabs
 
 COPY ./dist/version                                             /home/ranger/dist/
 COPY ./dist/ranger-${HBASE_PLUGIN_VERSION}-hbase-plugin.tar.gz  /home/ranger/dist/
 COPY ./downloads/hbase-${HBASE_VERSION}-bin.tar.gz              /home/ranger/dist/
 
-COPY ./scripts/ranger-hbase-setup.sh                     /home/ranger/scripts/
-COPY ./scripts/ranger-hbase.sh                           /home/ranger/scripts/
-COPY ./scripts/ranger-hbase-plugin-install.properties    /home/ranger/scripts/
-COPY ./scripts/hbase-site.xml                            /home/ranger/scripts/
+COPY ./scripts/hbase/ranger-hbase-setup.sh                  ${RANGER_SCRIPTS}/
+COPY ./scripts/hbase/ranger-hbase.sh                        ${RANGER_SCRIPTS}/
+COPY ./scripts/hbase/ranger-hbase-plugin-install.properties ${RANGER_SCRIPTS}/
+COPY ./scripts/hbase/hbase-site.xml                         ${RANGER_SCRIPTS}/
+COPY ./scripts/hadoop/core-site.xml                         ${RANGER_SCRIPTS}/
+COPY ./scripts/wait_for_keytab.sh                           ${RANGER_SCRIPTS}/
+COPY ./scripts/kdc/krb5.conf                                /etc/krb5.conf
 
 RUN tar xvfz /home/ranger/dist/hbase-${HBASE_VERSION}-bin.tar.gz --directory=/opt/ && \
     ln -s /opt/hbase-${HBASE_VERSION} /opt/hbase && \
     rm -f /home/ranger/dist/hbase-${HBASE_VERSION}-bin.tar.gz && \
     tar xvfz /home/ranger/dist/ranger-${HBASE_PLUGIN_VERSION}-hbase-plugin.tar.gz --directory=/opt/ranger && \
     ln -s /opt/ranger/ranger-${HBASE_PLUGIN_VERSION}-hbase-plugin /opt/ranger/ranger-hbase-plugin && \
     rm -f /home/ranger/dist/ranger-${HBASE_PLUGIN_VERSION}-hbase-plugin.tar.gz && \
-    cp -f /home/ranger/scripts/ranger-hbase-plugin-install.properties /opt/ranger/ranger-hbase-plugin/install.properties && \
+    cp -f ${RANGER_SCRIPTS}/ranger-hbase-plugin-install.properties /opt/ranger/ranger-hbase-plugin/install.properties && \
+    chmod 755 ${RANGER_SCRIPTS}/wait_for_keytab.sh && \
     chmod 744 ${RANGER_SCRIPTS}/ranger-hbase-setup.sh ${RANGER_SCRIPTS}/ranger-hbase.sh
 
 RUN apt-get update && \

dev-support/ranger-docker/Dockerfile.ranger-hive

@@ -25,6 +25,7 @@ ARG HIVE_PLUGIN_VERSION
 ARG TEZ_VERSION
 ARG RANGER_DB_TYPE
 
+VOLUME /etc/keytabs
 
 COPY ./dist/version                                          /home/ranger/dist/
 COPY ./dist/ranger-${HIVE_PLUGIN_VERSION}-hive-plugin.tar.gz /home/ranger/dist/
@@ -35,10 +36,13 @@ COPY ./downloads/postgresql-42.2.16.jre7.jar                 /home/ranger/dist/
 COPY ./downloads/mysql-connector-java-8.0.28.jar             /home/ranger/dist/
 COPY ./downloads/ojdbc8.jar                                  /home/ranger/dist/
 
-COPY ./scripts/ranger-hive-setup.sh                     /home/ranger/scripts/
-COPY ./scripts/ranger-hive.sh                           /home/ranger/scripts/
-COPY ./scripts/ranger-hive-plugin-install.properties    /home/ranger/scripts/
-COPY ./scripts/hive-site-${RANGER_DB_TYPE}.xml          /home/ranger/scripts/hive-site.xml
+COPY ./scripts/hive/ranger-hive-setup.sh                  ${RANGER_SCRIPTS}/
+COPY ./scripts/hive/ranger-hive.sh                        ${RANGER_SCRIPTS}/
+COPY ./scripts/hive/ranger-hive-plugin-install.properties ${RANGER_SCRIPTS}/
+COPY ./scripts/hive/hive-site-${RANGER_DB_TYPE}.xml       ${RANGER_SCRIPTS}/hive-site.xml
+COPY ./scripts/hadoop/core-site.xml                       ${RANGER_SCRIPTS}/
+COPY ./scripts/wait_for_keytab.sh                         ${RANGER_SCRIPTS}/
+COPY ./scripts/kdc/krb5.conf                              /etc/krb5.conf
 
 RUN cd /opt && tar xzf /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz && \
     ln -s /opt/apache-hive-${HIVE_VERSION}-bin /opt/hive && \
@@ -55,7 +59,8 @@ RUN cd /opt && tar xzf /home/ranger/dist/apache-hive-${HIVE_VERSION}-bin.tar.gz
     tar xvfz /home/ranger/dist/ranger-${HIVE_PLUGIN_VERSION}-hive-plugin.tar.gz --directory=/opt/ranger && \
     ln -s /opt/ranger/ranger-${HIVE_PLUGIN_VERSION}-hive-plugin /opt/ranger/ranger-hive-plugin && \
     rm -f /home/ranger/dist/ranger-${HIVE_PLUGIN_VERSION}-hive-plugin.tar.gz && \
-    cp -f /home/ranger/scripts/ranger-hive-plugin-install.properties /opt/ranger/ranger-hive-plugin/install.properties && \
+    cp -f ${RANGER_SCRIPTS}/ranger-hive-plugin-install.properties /opt/ranger/ranger-hive-plugin/install.properties && \
+    chmod 755 ${RANGER_SCRIPTS}/wait_for_keytab.sh && \
     chmod 744 ${RANGER_SCRIPTS}/ranger-hive-setup.sh ${RANGER_SCRIPTS}/ranger-hive.sh
 
 ENV HIVE_HOME=/opt/hive

dev-support/ranger-docker/Dockerfile.ranger-kafka

@@ -21,25 +21,30 @@ FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION}
 ARG KAFKA_VERSION
 ARG KAFKA_PLUGIN_VERSION
 
+VOLUME /etc/keytabs
 
 COPY ./dist/version                                            /home/ranger/dist/
 COPY ./dist/ranger-${KAFKA_PLUGIN_VERSION}-kafka-plugin.tar.gz /home/ranger/dist/
 COPY ./downloads/kafka_2.12-${KAFKA_VERSION}.tgz               /home/ranger/dist/
 
-COPY ./scripts/ranger-kafka-setup.sh                     /home/ranger/scripts/
-COPY ./scripts/ranger-kafka.sh                           /home/ranger/scripts/
-COPY ./scripts/ranger-kafka-plugin-install.properties    /home/ranger/scripts/
+COPY ./scripts/kafka/ranger-kafka-setup.sh                  ${RANGER_SCRIPTS}/
+COPY ./scripts/kafka/ranger-kafka.sh                        ${RANGER_SCRIPTS}/
+COPY ./scripts/kafka/ranger-kafka-plugin-install.properties ${RANGER_SCRIPTS}/
+COPY ./scripts/kafka/kafka-server-jaas.conf                 ${RANGER_SCRIPTS}/
+COPY ./scripts/hadoop/core-site.xml                         ${RANGER_SCRIPTS}/
+COPY ./scripts/wait_for_keytab.sh                           ${RANGER_SCRIPTS}/
+COPY ./scripts/kdc/krb5.conf                                /etc/krb5.conf
 
 RUN tar xvfz /home/ranger/dist/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/ && \
     ln -s /opt/kafka_2.12-${KAFKA_VERSION} /opt/kafka && \
     rm -f /home/ranger/dist/kafka_2.12-${KAFKA_VERSION}.tgz && \
     tar xvfz /home/ranger/dist/ranger-${KAFKA_PLUGIN_VERSION}-kafka-plugin.tar.gz --directory=/opt/ranger && \
     ln -s /opt/ranger/ranger-${KAFKA_PLUGIN_VERSION}-kafka-plugin /opt/ranger/ranger-kafka-plugin && \
     rm -f /home/ranger/dist/ranger-${KAFKA_PLUGIN_VERSION}-kafka-plugin.tar.gz && \
-    cp -f /home/ranger/scripts/ranger-kafka-plugin-install.properties /opt/ranger/ranger-kafka-plugin/install.properties && \
+    cp -f ${RANGER_SCRIPTS}/ranger-kafka-plugin-install.properties /opt/ranger/ranger-kafka-plugin/install.properties && \
+    chmod 755 ${RANGER_SCRIPTS}/wait_for_keytab.sh && \
     chmod 744 ${RANGER_SCRIPTS}/ranger-kafka-setup.sh ${RANGER_SCRIPTS}/ranger-kafka.sh
 
-
 ENV KAFKA_HOME=/opt/kafka
 ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/kafka/bin
 

dev-support/ranger-docker/Dockerfile.ranger-kdc

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG RANGER_BASE_JAVA_VERSION=8
+
+FROM eclipse-temurin:${RANGER_BASE_JAVA_VERSION}-jdk-jammy
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV REALM=EXAMPLE.COM
+ENV KDC_HOST=kdc.example.com
+ENV ADMIN_PRINCIPAL=admin/admin
+ENV ADMIN_PASSWORD=rangerR0cks!
+ENV MASTER_PASSWORD=rangerR0cks!
+
+# Install Kerberos components
+RUN apt-get update && \
+    apt-get install -y krb5-kdc krb5-admin-server krb5-user && \
+    rm -rf /var/lib/apt/lists/*
+
+# Copy configuration files
+COPY ./scripts/kdc/krb5.conf     /etc/krb5.conf
+COPY ./scripts/kdc/kdc.conf      /etc/krb5kdc/kdc.conf
+COPY ./scripts/kdc/kadm5.acl     /etc/krb5kdc/kadm5.acl
+COPY ./scripts/kdc/entrypoint.sh /entrypoint.sh
+
+RUN chmod +x /entrypoint.sh
+
+VOLUME /etc/keytabs
+
+EXPOSE 88/tcp 88/udp 749/tcp
+
+ENTRYPOINT ["/entrypoint.sh"]

dev-support/ranger-docker/Dockerfile.ranger-kms

@@ -22,11 +22,16 @@ FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION} AS ranger-kms
 ARG KMS_VERSION
 ARG RANGER_DB_TYPE
 
+VOLUME /etc/keytabs
+
 COPY ./dist/version                                            /home/ranger/dist/
 COPY ./dist/ranger-${KMS_VERSION}-kms.tar.gz                   /home/ranger/dist/
 
-COPY ./scripts/ranger-kms.sh                                   ${RANGER_SCRIPTS}/
-COPY ./scripts/ranger-kms-install-${RANGER_DB_TYPE}.properties ${RANGER_SCRIPTS}/ranger-kms-install.properties
+COPY ./scripts/kms/ranger-kms.sh                                   ${RANGER_SCRIPTS}/
+COPY ./scripts/kms/ranger-kms-install-${RANGER_DB_TYPE}.properties ${RANGER_SCRIPTS}/ranger-kms-install.properties
+COPY ./scripts/hadoop/core-site.xml                                ${RANGER_SCRIPTS}/
+COPY ./scripts/wait_for_keytab.sh                                  ${RANGER_SCRIPTS}/
+COPY ./scripts/kdc/krb5.conf                                       /etc/krb5.conf
 
 RUN tar xvfz /home/ranger/dist/ranger-${KMS_VERSION}-kms.tar.gz --directory=${RANGER_HOME} && \
     ln -s ${RANGER_HOME}/ranger-${KMS_VERSION}-kms ${RANGER_HOME}/kms && \
@@ -40,6 +45,7 @@ RUN tar xvfz /home/ranger/dist/ranger-${KMS_VERSION}-kms.tar.gz --directory=${RA
     ln -s /etc/init.d/ranger-kms /etc/rc3.d/K90ranger-kms && \
     ln -s ${RANGER_HOME}/kms/ranger-kms-services.sh /usr/bin/ranger-kms-services.sh && \
     chown -R rangerkms:ranger ${RANGER_HOME}/kms/ ${RANGER_SCRIPTS}/ /var/run/ranger_kms/ /var/log/ranger/ /etc/ranger && \
+    chmod 755 ${RANGER_SCRIPTS}/wait_for_keytab.sh && \
     chmod 744 ${RANGER_SCRIPTS}/ranger-kms.sh
 
 FROM ranger-kms AS ranger_postgres