HDDS-9266. Acceptance test for snapshot data validation after OM bootstrapping

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config-ratis-om-bootstrap

@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE-SITE.XML_ozone.om.ratis.log.purge.gap=50
+OZONE-SITE.XML_ozone.om.ratis.segment.size=16KB
+OZONE-SITE.XML_ozone.om.ratis.segment.preallocated.size=16KB
+OZONE-SITE.XML_ozone.om.ratis.snapshot.auto.trigger.threshold=500

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/om-bootstrap.yaml

@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.8"
+
+x-OM-Ratis-config:
+  &common-env-file
+  env_file:
+    - ./docker-config
+    - ./docker-config-ratis-om-bootstrap
+
+x-om3-setup:
+  &om3-setup
+  environment:
+    WAITFOR:
+    # Skip initialising the OM, so that the container doesn't have any data.
+    ENSURE_OM_INITIALIZED:
+    OZONE_OPTS:
+  # This command produces the same behavior as sleeping indefinitely.
+  command: [ "tail","-f","/dev/null" ]
+
+services:
+  kdc:
+    <<: *common-env-file
+  kms:
+    <<: *common-env-file
+  datanode1:
+    <<: *common-env-file
+  datanode2:
+    <<: *common-env-file
+  datanode3:
+    <<: *common-env-file
+  om1:
+    <<: *common-env-file
+  om2:
+    <<: *common-env-file
+  om3:
+    <<: [*om3-setup, *common-env-file]
+  httpfs:
+    <<: *common-env-file
+  s3g:
+    <<: *common-env-file
+  scm1.org:
+    <<: *common-env-file
+  scm2.org:
+    <<: *common-env-file
+  scm3.org:
+    <<: *common-env-file
+  recon:
+    <<: *common-env-file

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-om-bootstrap.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#suite:HA-secure
+
+# This test aims to validate the ozone snapshot data that have been
+# installed on a bootstrapped OM after a Ratis snapshot installation.
+#
+# The test
+#   * starts the docker environment with 'om3' inactive and uninitialised
+#   * runs a robot test that creates keys and snapshots
+#   * checks that 'om3' is inactive and has no data
+#   * initialises 'om3'
+#   * starts 'om3'
+#   * verifies that 'om3' is running and is bootstrapping
+#   * runs a robot test that validates the data on 'om3'
+#
+# The data creation robot test
+#   * creates 100 metadata keys
+#   * creates the first snapshot
+#   * creates two actual keys and set the contents of each key, the same as the key name
+#   * creates the second snapshot
+#
+# The data validation robot test
+#   * checks that there have been checkpoints created on 'om3'
+#   * once checkpoints are created, the 'om3' has all the data from the leader
+#   * checks that 'om3' is not leader
+#   * transfers leadership to 'om3', so that we can perform regular leader reads
+#   * checks that the two snapshots exist on 'om3'
+#   * runs a snapshot diff between the two snapshots
+#   * validates that the result of the snapshot diff, contains just the two actual keys
+#   * does a 'key cat' on both snapshot keys and validates the contents
+#   * the keys are read from the snapshot and not the active file system
+#   * the contents of each key should be the same as the key name
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+export SECURITY_ENABLED=true
+export OM_SERVICE_ID="omservice"
+export SCM=scm1.org
+export COMPOSE_FILE=docker-compose.yaml:om-bootstrap.yaml
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../testlib.sh"
+
+start_docker_env
+
+volume="vol1"
+bucket="bucket1"
+snap1="snap1"
+snap2="snap2"
+keyPrefix="sn"
+key1="key1"
+key2="key2"
+bootstrap_om="om3"
+
+execute_robot_test om1 kinit.robot
+
+# Data creation
+execute_robot_test om1 -v VOLUME:${volume} -v BUCKET:${bucket} -v SNAP_1:${snap1} -v SNAP_2:${snap2} -v KEY_PREFIX:${keyPrefix} -v KEY_1:${key1} -v KEY_2:${key2} omha/data-creation-before-om-bootstrap.robot
+
+echo "Check that om3 isn't running"
+om3_service=$(execute_command_in_container om3 ps aux | grep 'OzoneManagerStarter' || true)
+
+if [[ $om3_service != "" ]]
+then
+  echo "om3 is running, exiting..."
+  exit 1
+fi
+
+echo "Check that om3 has no data"
+om3_data=$(execute_command_in_container om3 ls -lah /data | grep 'metadata' || true)
+
+if [[ $om3_data != "" ]]
+then
+  echo "om3 has data, exiting..."
+  exit 1
+fi
+
+# Init om3 and start the om daemon in the background
+execute_command_in_container om3 ozone om --init
+execute_command_in_container -d om3 ozone om
+wait_for_port om3 9872 120
+
+echo "Check that om3 is running"
+om3_service=$(execute_command_in_container om3 ps aux | grep 'OzoneManagerStarter' || true)
+
+if [[ $om3_service == "" ]]
+then
+  echo "om3 isn't running, exiting..."
+  exit 1
+fi
+
+echo "Check that om3 has data"
+om3_data=$(execute_command_in_container om3 ls -lah /data | grep 'metadata' || true)
+
+if [[ $om3_data == "" ]]
+then
+  echo "om3 has no data, exiting..."
+  exit 1
+fi
+
+execute_robot_test om3 kinit.robot
+
+# This test checks the disk on the node it's running. It needs to be run on om3.
+execute_robot_test om3 -v BOOTSTRAPPED_OM:${bootstrap_om} -v VOLUME:${volume} -v BUCKET:${bucket} -v SNAP_1:${snap1} -v SNAP_2:${snap2} -v KEY_PREFIX:${keyPrefix} -v KEY_1:${key1} -v KEY_2:${key2} omha/data-validation-after-om-bootstrap.robot

hadoop-ozone/dist/src/main/smoketest/commonlib.robot

@@ -28,7 +28,7 @@ ${OM_SERVICE_ID}     om
 *** Keywords ***
 Get test user principal
     [arguments]         ${user}
-    ${instance} =       Execute                    hostname | sed 's/scm[0-9].org/scm/' | sed 's/om[0-9]/om/'
+    ${instance} =       Execute                    hostname | sed 's/scm[0-9].org/scm/;s/scm[0-9]/scm/;s/om[0-9]/om/'
     [return]            ${user}/${instance}@EXAMPLE.COM
 
 Kinit HTTP user

hadoop-ozone/dist/src/main/smoketest/omha/data-creation-before-om-bootstrap.robot

@@ -0,0 +1,96 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Smoke test for creating data needed for om bootstrap load test.
+Resource            ../commonlib.robot
+Test Timeout        5 minutes
+Test Setup          Run Keyword if    '${SECURITY_ENABLED}' == 'true'    Kinit test user     testuser     testuser.keytab
+
+*** Variables ***
+${TMP_FILE}         tmp.txt
+${VOLUME}
+${BUCKET}
+${SNAP_1}
+${SNAP_2}
+${KEY_PREFIX}
+${KEY_1}
+${KEY_2}
+
+*** Keywords ***
+Create volume and bucket
+    [arguments]         ${volume}           ${bucket}
+    ${vol_res} =        Execute             ozone sh volume create /${volume}
+                        Should Be Empty     ${vol_res}
+    ${bucket_res} =     Execute             ozone sh bucket create /${volume}/${bucket}
+                        Should Be Empty     ${bucket_res}
+
+Create tmp file
+    [arguments]         ${file_name}
+    ${create_res} =     Execute             touch ${file_name}
+                        Should Be Empty     ${create_res}
+    ${ls_grep_res} =    Execute             ls -lah | grep '${file_name}'
+                        Should contain      ${ls_grep_res}      ${file_name}
+
+Delete tmp file
+    [arguments]         ${file_name}
+    Execute             rm ${file_name}
+    ${file_exists} =    Execute             [[ -f ${file_name} ]] && echo "File exists" || echo "File doesn't exist"
+                        Should contain      ${file_exists}      File doesn't exist
+
+Create a key and set contents same as the keyName
+    [arguments]         ${volume}           ${bucket}      ${key_prefix}       ${key_name}         ${tmp_file}
+    Execute             echo "${key_prefix}/${key_name}" > ${tmp_file}
+    ${key_res} =        Execute             ozone sh key put /${volume}/${bucket}/${key_prefix}/${key_name} ${tmp_file}
+                        Should Be Empty     ${key_res}
+    ${key_cat_res} =    Execute             ozone sh key cat /${volume}/${bucket}/${key_prefix}/${key_name}
+                        Should contain      ${key_cat_res}      ${key_prefix}/${key_name}
+
+Create actual keys
+    [arguments]         ${volume}           ${bucket}           ${key_prefix}       ${key_1}            ${key2}             ${tmp_file}
+    Create a key and set contents same as the keyName           ${volume}           ${bucket}           ${key_prefix}       ${key_1}            ${tmp_file}
+    Create a key and set contents same as the keyName           ${volume}           ${bucket}           ${key_prefix}       ${key_2}            ${tmp_file}
+
+Create metadata keys
+    [arguments]         ${threads}          ${key_num}          ${volume}       ${bucket}
+    ${freon_res} =      Execute             ozone freon omkg -t ${threads} -n ${key_num} -v ${volume} -b ${bucket}
+                        Should contain      ${freon_res}        Successful executions: ${key_num}
+
+Create snapshot
+    [arguments]         ${volume}           ${bucket}       ${snapshot}
+    ${snap_res} =       Execute             ozone sh snapshot create /${volume}/${bucket} ${snapshot}
+                        Should Be Empty     ${snap_res}
+
+*** Test Cases ***
+Volume-bucket init
+    Create volume and bucket        ${VOLUME}       ${BUCKET}
+
+Create 100 metadata keys under /${VOLUME}/${BUCKET}
+    Create metadata keys     10      100             ${VOLUME}       ${BUCKET}
+
+Create snapshot '${SNAP_1}'
+    Create snapshot         ${VOLUME}               ${BUCKET}       ${SNAP_1}
+
+Create tmp file to be used for key creation
+    Create tmp file         ${TMP_FILE}
+
+Create 2 actual keys with prefix '${KEY_PREFIX}', key contents the same as the key name
+    Create actual keys      ${VOLUME}               ${BUCKET}       ${KEY_PREFIX}       ${KEY_1}        ${KEY_2}        ${TMP_FILE}
+
+Create snapshot '${SNAP_2}'
+    Create snapshot         ${VOLUME}               ${BUCKET}       ${SNAP_2}
+
+Cleanup tmp file
+    Delete tmp file         ${TMP_FILE}