Issue #2955 : Warn when a referenced database connection does not exist

core/src/main/java/org/apache/hop/core/util/StringUtil.java

@@ -579,6 +579,24 @@ public static boolean isVariable(String variable) {
         || variable.startsWith(HEX_OPEN) && variable.endsWith(HEX_CLOSE);
   }
 
+  /**
+   * Whether {@code value} still contains a Hop variable delimiter after substitution. Used to skip
+   * design-time checks that cannot be decided when a name or URL still holds {@code ${...}}, {@code
+   * %%...%%}, {@code $[...]} or {@code #{...}}.
+   *
+   * @param value the string to inspect, may be null
+   * @return true when a variable token is still present
+   */
+  public static boolean containsVariableToken(String value) {
+    if (value == null) {
+      return false;
+    }
+    return value.contains(UNIX_OPEN)
+        || value.contains(WINDOWS_OPEN)
+        || value.contains(HEX_OPEN)
+        || value.contains(RESOLVER_OPEN);
+  }
+
   /**
    * Calls the {@link String#toLowerCase()} method on the {@link String} returned by a call to
    * {@code obj.toString()}, guarding against {@link NullPointerException}s.

core/src/main/java/org/apache/hop/metadata/util/HopMetadataPropertyWalker.java

@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.metadata.util;
+
+import java.lang.reflect.Array;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
+
+/**
+ * Walks {@link HopMetadataProperty} fields on a metadata object, including nested objects and
+ * collections, and collects string values of a given {@link HopMetadataPropertyType}.
+ *
+ * <p>Failures to read a field are skipped. Cycles are broken. This is a design-time helper: it must
+ * not throw because of a broken plugin class.
+ */
+public final class HopMetadataPropertyWalker {
+
+  private static final int MAX_DEPTH = 8;
+
+  private HopMetadataPropertyWalker() {}
+
+  /**
+   * A string property found on a metadata object.
+   *
+   * @param type the annotated property type
+   * @param key the serialised key, or the field name when no key is set
+   * @param value the raw (unresolved) string value, never null
+   */
+  public record StringProperty(HopMetadataPropertyType type, String key, String value) {}
+
+  /**
+   * Collect every string field annotated with {@code type} under {@code root}.
+   *
+   * @param root the object to walk, may be null
+   * @param type the property type to collect
+   * @return the matching properties, possibly empty
+   */
+  public static List<StringProperty> collectStrings(Object root, HopMetadataPropertyType type) {
+    List<StringProperty> collected = new ArrayList<>();
+    if (root == null || type == null) {
+      return collected;
+    }
+    walk(
+        root,
+        type,
+        collected,
+        0,
+        java.util.Collections.newSetFromMap(new IdentityHashMap<Object, Boolean>()));
+    return collected;
+  }
+
+  private static void walk(
+      Object node,
+      HopMetadataPropertyType type,
+      List<StringProperty> collected,
+      int depth,
+      Set<Object> visited) {
+    if (node == null || depth > MAX_DEPTH || !isMetadataObject(node) || !visited.add(node)) {
+      return;
+    }
+    for (Field field : ReflectionUtil.findAllFields(node.getClass())) {
+      if (Modifier.isStatic(field.getModifiers())) {
+        continue;
+      }
+      HopMetadataProperty property = field.getAnnotation(HopMetadataProperty.class);
+      if (property == null) {
+        continue;
+      }
+      Object value = readField(field, node);
+      if (value == null) {
+        continue;
+      }
+      if (property.hopMetadataPropertyType() == type && value instanceof String stringValue) {
+        collected.add(new StringProperty(type, serialisedKey(property, field), stringValue));
+      }
+      descend(value, type, collected, depth, visited);
+    }
+  }
+
+  private static void descend(
+      Object value,
+      HopMetadataPropertyType type,
+      List<StringProperty> collected,
+      int depth,
+      Set<Object> visited) {
+    if (value instanceof Collection<?> collection) {
+      for (Object element : collection) {
+        walk(element, type, collected, depth + 1, visited);
+      }
+      return;
+    }
+    if (value instanceof Map<?, ?> map) {
+      for (Object element : map.values()) {
+        walk(element, type, collected, depth + 1, visited);
+      }
+      return;
+    }
+    if (value.getClass().isArray()) {
+      int length = Array.getLength(value);
+      for (int i = 0; i < length; i++) {
+        walk(Array.get(value, i), type, collected, depth + 1, visited);
+      }
+      return;
+    }
+    walk(value, type, collected, depth + 1, visited);
+  }
+
+  private static String serialisedKey(HopMetadataProperty property, Field field) {
+    if (property.key() != null && !property.key().isEmpty()) {
+      return property.key();
+    }
+    return field.getName();
+  }
+
+  /** Only descends into Hop's own metadata classes, never into JDK or third-party types. */
+  static boolean isMetadataObject(Object value) {
+    if (value == null) {
+      return false;
+    }
+    Class<?> type = value.getClass();
+    if (type.isPrimitive() || type.isEnum() || type.isArray()) {
+      return false;
+    }
+    Package pkg = type.getPackage();
+    return pkg != null && pkg.getName().toLowerCase(Locale.ROOT).startsWith("org.apache.hop");
+  }
+
+  private static Object readField(Field field, Object target) {
+    try {
+      field.setAccessible(true);
+      return field.get(target);
+    } catch (Exception e) {
+      return null;
+    }
+  }
+}

core/src/test/java/org/apache/hop/core/util/StringUtilTest.java

@@ -162,6 +162,18 @@ void testIsVariable() {
     assertFalse(StringUtil.isVariable(null));
   }
 
+  @Test
+  void testContainsVariableToken() {
+    assertTrue(StringUtil.containsVariableToken("${CONNECTION}"));
+    assertTrue(StringUtil.containsVariableToken("db_${ENV}"));
+    assertTrue(StringUtil.containsVariableToken("%%WINDOWS%%"));
+    assertTrue(StringUtil.containsVariableToken("$[hex]"));
+    assertTrue(StringUtil.containsVariableToken("#{resolver}"));
+    assertFalse(StringUtil.containsVariableToken("sales-db"));
+    assertFalse(StringUtil.containsVariableToken(null));
+    assertFalse(StringUtil.containsVariableToken(""));
+  }
+
   @Test
   void testSafeToLowerCase() {
     assertNull(StringUtil.safeToLowerCase(null));

core/src/test/java/org/apache/hop/metadata/util/HopMetadataPropertyWalkerTest.java

@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.metadata.util;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
+import org.apache.hop.metadata.util.HopMetadataPropertyWalker.StringProperty;
+import org.junit.jupiter.api.Test;
+
+class HopMetadataPropertyWalkerTest {
+
+  static class SimpleMeta {
+    @HopMetadataProperty(hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_CONNECTION)
+    String connection = "warehouse";
+
+    @HopMetadataProperty(key = "sql")
+    String sql = "SELECT 1";
+
+    String unannotated = "ignored";
+  }
+
+  static class NestedItem {
+    @HopMetadataProperty(
+        key = "name",
+        hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_CONNECTION)
+    String name;
+
+    NestedItem(String name) {
+      this.name = name;
+    }
+  }
+
+  static class NestedMeta {
+    @HopMetadataProperty(hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_CONNECTION)
+    String connection = "primary";
+
+    @HopMetadataProperty
+    List<NestedItem> items = List.of(new NestedItem("second"), new NestedItem("third"));
+  }
+
+  static class TwoConnectionsMeta {
+    @HopMetadataProperty(
+        key = "referenceConnection",
+        hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_CONNECTION)
+    String reference = "ref-db";
+
+    @HopMetadataProperty(
+        key = "compareConnection",
+        hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_CONNECTION)
+    String compare = "cmp-db";
+  }
+
+  static class UnannotatedConnectionMeta {
+    @HopMetadataProperty(key = "connection")
+    String connection = "hidden";
+  }
+
+  @Test
+  void collectsAnnotatedConnectionStrings() {
+    List<StringProperty> found =
+        HopMetadataPropertyWalker.collectStrings(
+            new SimpleMeta(), HopMetadataPropertyType.RDBMS_CONNECTION);
+
+    assertEquals(1, found.size());
+    assertEquals("connection", found.get(0).key());
+    assertEquals("warehouse", found.get(0).value());
+  }
+
+  @Test
+  void descendsIntoNestedLists() {
+    List<StringProperty> found =
+        HopMetadataPropertyWalker.collectStrings(
+            new NestedMeta(), HopMetadataPropertyType.RDBMS_CONNECTION);
+
+    assertEquals(3, found.size());
+    assertEquals(
+        List.of("primary", "second", "third"), found.stream().map(StringProperty::value).toList());
+  }
+
+  @Test
+  void collectsTwoConnectionFieldsOnOneObject() {
+    List<StringProperty> found =
+        HopMetadataPropertyWalker.collectStrings(
+            new TwoConnectionsMeta(), HopMetadataPropertyType.RDBMS_CONNECTION);
+
+    assertEquals(2, found.size());
+    assertTrue(found.stream().anyMatch(p -> "referenceConnection".equals(p.key())));
+    assertTrue(found.stream().anyMatch(p -> "compareConnection".equals(p.key())));
+  }
+
+  @Test
+  void ignoresConnectionFieldsWithoutThePropertyType() {
+    List<StringProperty> found =
+        HopMetadataPropertyWalker.collectStrings(
+            new UnannotatedConnectionMeta(), HopMetadataPropertyType.RDBMS_CONNECTION);
+
+    assertTrue(found.isEmpty());
+  }
+
+  @Test
+  void nullRootYieldsNothing() {
+    assertTrue(
+        HopMetadataPropertyWalker.collectStrings(null, HopMetadataPropertyType.RDBMS_CONNECTION)
+            .isEmpty());
+  }
+}

docs/hop-user-manual/modules/ROOT/pages/hop-gui/perspective-configuration.adoc

@@ -138,6 +138,13 @@ image::hop-gui/configuration-perspective-open-help-pages-in.png[Open help pages
 * xref:vfs/google-drive-vfs.adoc#_configuration[Google Drive] VFS configuration options.
 * xref:projects/index.adoc[Project] configuration options
 * Welcome Dialog: specify whether to show or hide the welcome dialog when Hop GUI starts.
+* **File validation**: when enabled (the default), saving a pipeline or workflow warns if a transform or action references a relational database connection that is not in the project metadata. You can still save. Connection names that still contain a variable such as `'${CONNECTION}'` after the current environment is applied are skipped, because the name cannot be decided at design time. The same check also runs when you Verify a pipeline or workflow. Hop does not try to open a JDBC connection as part of this check.
++
+image::hop-gui/configuration-perspective-file-validation-validate-database-connections-when-saving.png[Validate database connections when saving,width="90%"]
++
+The warning lists the missing connections. *Yes* saves anyway, *No* cancels the save. Check *Don't run this check when saving* to turn the option off; you can turn it back on here.
++
+image::hop-gui/validate-database-connections-on-save-warning-dialog.png[Database connections save warning,width="90%"]
 
 === System Variables
 

docs/hop-user-manual/modules/ROOT/pages/metadata-types/rdbms-connection.adoc

@@ -30,6 +30,10 @@ Hop supports tens of relational databases out of the box. If your specific datab
 
 Check the list of xref:database/databases.adoc[databases] for more details.
 
+Transforms and actions store the *name* of a relational database connection. When you save or Verify a pipeline or workflow, Hop warns if that name is not in the project metadata. You can still save. If the name still contains a variable such as `'${CONNECTION}'` after the current environment is applied, the check is skipped because the name cannot be decided at design time. This does not open a JDBC connection; it only looks the name up in metadata. The warning dialog can turn the check off, or you can disable it under xref:hop-gui/perspective-configuration.adoc[Configuration] → Plugins → File validation.
+
+image::hop-gui/validate-database-connections-on-save-warning-dialog.png[Database connections save warning,width="90%"]
+
 
 == Related Plugins
 

docs/hop-user-manual/modules/ROOT/pages/pipeline/create-pipeline.adoc

@@ -61,6 +61,10 @@ When you are finished with your pipeline, save it.
 This can be done via the File menu, the icons or using CTLR s or Command s.
 For new pipelines a file browser is displayed to navigate towards the location you want to store the file.
 
+By default Hop warns if a transform references a relational database connection that is not in the project metadata. You can still save. Connection names that still contain a variable such as `'${CONNECTION}'` are skipped. Check *Don't run this check when saving* on the dialog, or turn the option off under xref:hop-gui/perspective-configuration.adoc[Configuration] → Plugins → File validation.
+
+image::hop-gui/validate-database-connections-on-save-warning-dialog.png[Database connections save warning,width="90%"]
+
 == Add Transform to your pipelines
 
 Click anywhere in the pipeline canvas, the area where you'll see the image below.

docs/hop-user-manual/modules/ROOT/pages/workflow/create-workflow.adoc

@@ -58,6 +58,10 @@ When you are finished with your workflow, save it.
 This can be done via the File menu, the icons or using CTLR s or Command s.
 For new workflows a file browser is displayed to navigate towards the location you want to store the file.
 
+By default Hop warns if an action references a relational database connection that is not in the project metadata. You can still save. Connection names that still contain a variable such as `'${CONNECTION}'` are skipped. Check *Don't run this check when saving* on the dialog, or turn the option off under xref:hop-gui/perspective-configuration.adoc[Configuration] → Plugins → File validation.
+
+image::hop-gui/validate-database-connections-on-save-warning-dialog.png[Database connections save warning,width="90%"]
+
 == Add Action to your workflow
 
 Add the following actions to your workflow and create the hops to connect them: