issue #7393 : Merge Rows layout alignment option for CDC

docker/integration-tests/integration-tests-base.yaml

@@ -31,4 +31,6 @@ services:
       - ../../integration-tests/:/files
     environment:
       - FLASK_ENV=docker
+      # Optional workflow basename filter (substring or glob). Set by run-tests-docker.sh.
+      - TEST_FILTER=${TEST_FILTER:-}
     command: [ "bash", "-c", "/files/scripts/run-tests.sh ${PROJECT_NAME}" ]

docker/integration-tests/integration-tests-beam-base.yaml

@@ -29,4 +29,6 @@ services:
       - ../../integration-tests/:/files
     environment:
       - FLASK_ENV=docker
+      # Optional workflow basename filter (substring or glob). Set by run-tests-docker.sh.
+      - TEST_FILTER=${TEST_FILTER:-}
     command: [ "bash", "-c", "/files/scripts/run-tests.sh ${PROJECT_NAME}" ]

docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/mergerows.adoc

@@ -78,11 +78,17 @@ In the subsequent transform, you can use the flag field generated by **Merge row
 |Flag field name|Specify the name of the flag field on the output stream.
 |Difference field name|Specify the name of the field to contain the difference in case the status is **changed**.
 The difference will be in the form of a "changes" array in JSON format.
+|Align input layouts|When enabled (the default), the transform builds a **reference-authoritative** union of the reference and compare field layouts before comparing.
+Missing fields on either side are filled with null.
+Shared fields keep the reference data type; compatible compare values are converted to that type.
+Compare-only fields are appended to the output layout.
+Disable this option to require identical layouts (legacy strict mode).
 |Keys to match|Specify fields containing the keys on which to match. Click "Get key fields" to insert all the fields from the reference rows
 |Values to compare|Specify fields containing the values to compare. Click "Get value fields" to insert all the fields from the compare rows.
 Key fields do not need to be repeated here.
 |===
 
+
 == Passing through fields
 
 It's possible to pass through fields from the reference or compare data streams.

integration-tests/scripts/run-tests-docker.sh

@@ -29,6 +29,7 @@ for ARGUMENT in "$@"; do
 
   case "$KEY" in
   PROJECT_NAME) PROJECT_NAME=${VALUE} ;;
+  TEST_FILTER) TEST_FILTER=${VALUE} ;;
   JENKINS_USER) JENKINS_USER=${VALUE} ;;
   JENKINS_UID) JENKINS_UID=${VALUE} ;;
   JENKINS_GROUP) JENKINS_GROUP=${VALUE} ;;
@@ -44,6 +45,14 @@ if [ -z "${PROJECT_NAME}" ]; then
   PROJECT_NAME="*"
 fi
 
+# Optional filter for main*.hwf basenames (substring or glob, comma-separated).
+# Passed into the test container as TEST_FILTER. Examples:
+#   TEST_FILTER=0077-merge-rows
+#   TEST_FILTER='*0077*'
+if [ -z "${TEST_FILTER}" ]; then
+  TEST_FILTER=""
+fi
+
 if [ -z "${JENKINS_USER}" ]; then
   JENKINS_USER="jenkins"
 fi
@@ -117,13 +126,17 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do
 
       # Check if specific compose exists
 
+      if [ -n "${TEST_FILTER}" ]; then
+        echo "TEST_FILTER: ${TEST_FILTER}"
+      fi
+
       if [ -f "${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml" ]; then
         echo "Project compose exists."
         EXECUTED_COMPOSE_FILES=("${EXECUTED_COMPOSE_FILES[@]}" "${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml")
-        PROJECT_NAME=${PROJECT_NAME} docker compose -f ${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml up --abort-on-container-exit
+        PROJECT_NAME=${PROJECT_NAME} TEST_FILTER=${TEST_FILTER} docker compose -f ${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml up --abort-on-container-exit
       else
         echo "Project compose does not exists."
-        PROJECT_NAME=${PROJECT_NAME} docker compose -f ${DOCKER_FILES_DIR}/integration-tests-base.yaml up --abort-on-container-exit
+        PROJECT_NAME=${PROJECT_NAME} TEST_FILTER=${TEST_FILTER} docker compose -f ${DOCKER_FILES_DIR}/integration-tests-base.yaml up --abort-on-container-exit
       fi
     fi
   fi

integration-tests/scripts/run-tests.sh

@@ -104,9 +104,60 @@ if [ -z "${PROJECT_NAME}" ]; then
   PROJECT_NAME="*"
 fi
 
+# Optional filter for main*.hwf workflows (substring or glob against basename).
+# Comma-separated list is supported. Examples:
+#   TEST_FILTER=0077-merge-rows
+#   TEST_FILTER='*0077*','*0078*'
+#   TEST_FILTER=main-0077-merge-rows.hwf
+if [ -z "${TEST_FILTER}" ]; then
+  TEST_FILTER=""
+fi
+
 #set global variables
 SPACER="==========================================="
 
+# Return 0 if the workflow file should run under the current TEST_FILTER.
+should_run_workflow() {
+  local file="$1"
+  local base
+  base=$(basename "$file")
+
+  if [ -z "${TEST_FILTER}" ]; then
+    return 0
+  fi
+
+  local old_ifs=$IFS
+  IFS=','
+  local pattern
+  # shellcheck disable=SC2086
+  for pattern in ${TEST_FILTER}; do
+    # trim whitespace
+    pattern="${pattern#"${pattern%%[![:space:]]*}"}"
+    pattern="${pattern%"${pattern##*[![:space:]]}"}"
+    [ -z "${pattern}" ] && continue
+
+    # No glob meta-characters: treat as basename substring
+    if [[ "${pattern}" != *[\*\?[]* ]]; then
+      case "${base}" in
+      *"${pattern}"*)
+        IFS=$old_ifs
+        return 0
+        ;;
+      esac
+    else
+      # Glob match against basename
+      case "${base}" in
+      ${pattern})
+        IFS=$old_ifs
+        return 0
+        ;;
+      esac
+    fi
+  done
+  IFS=$old_ifs
+  return 1
+}
+
 # Set up a temporary folder
 export TMP_FOLDER=/tmp/hop-it-$$
 rm -rf "${TMP_FOLDER}"
@@ -161,8 +212,16 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do
       #
       # TODO: add hpl support when result is returned correctly
       #
+      if [ -n "${TEST_FILTER}" ]; then
+        echo "TEST_FILTER is set: ${TEST_FILTER}"
+      fi
+
       find $d -name 'main*.hwf' | sort | while read f; do
 
+        if ! should_run_workflow "$f"; then
+          continue
+        fi
+
         #cleanup temp files
         rm -f /tmp/test_output
         rm -f /tmp/test_output_err