fix(QTDI-2031): manage dynamic dependencies - w/o MAVEN-INF

component-api/src/main/java/org/talend/sdk/component/api/configuration/type/DynamicDependenciesConfiguration.java

@@ -0,0 +1,34 @@
+/**
+ * Copyright (C) 2006-2025 Talend Inc. - www.talend.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.talend.sdk.component.api.configuration.type;
+
+import static java.lang.annotation.ElementType.TYPE;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import org.talend.sdk.component.api.configuration.type.meta.ConfigurationType;
+import org.talend.sdk.component.api.meta.Documentation;
+
+@Target(TYPE)
+@Retention(RUNTIME)
+@ConfigurationType("dynamicDependenciesConfiguration")
+@Documentation("Mark a model (complex object) as being the configuration expected to compute dynamic dependencies.")
+public @interface DynamicDependenciesConfiguration {
+
+    String value() default "default";
+}

component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/ComponentManager.java

@@ -37,11 +37,9 @@
 import static org.talend.sdk.component.runtime.manager.reflect.Constructors.findConstructor;
 import static org.talend.sdk.component.runtime.manager.util.Lazy.lazy;
 
-import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.io.ObjectStreamException;
 import java.io.Serializable;
 import java.lang.annotation.Annotation;
@@ -1304,42 +1302,16 @@ public void onCreate(final Container container) {
             try {
                 String alreadyScannedClasses = null;
                 Filter filter = KnownClassesFilter.INSTANCE;
-                // we need to scan the nested repository
-                if (container.hasNestedRepository()) {
-                    try {
-                        final Enumeration<URL> urls = loader.getResources("TALEND-INF/scanning.properties");
+                try (final InputStream containerFilterConfig =
+                        container.getLoader().getResourceAsStream("TALEND-INF/scanning.properties")) {
+                    if (containerFilterConfig != null) {
                         final Properties config = new Properties();
-                        while (urls.hasMoreElements()) {
-                            final URL url = urls.nextElement();
-                            // ensure we scan the correct classes of plugin in the nested repository
-                            if ("nested".equals(url.getProtocol())
-                                    && url.getPath().contains(container.getRootModule())) {
-                                try (final BufferedReader reader =
-                                        new BufferedReader(new InputStreamReader(url.openStream()))) {
-                                    config.load(reader);
-                                    filter = createScanningFilter(config);
-                                    alreadyScannedClasses = config.getProperty("classes.list");
-                                    break;
-                                }
-                            }
-                        }
-                    } catch (IOException e) {
-                        log.info("[onCreate] Can't read nested scanning.properties: {}", e.getMessage());
-                    }
-                }
-                // normal scanning or nested scan failed
-                if (alreadyScannedClasses == null) {
-                    try (final InputStream containerFilterConfig =
-                            container.getLoader().getResourceAsStream("TALEND-INF/scanning.properties")) {
-                        if (containerFilterConfig != null) {
-                            final Properties config = new Properties();
-                            config.load(containerFilterConfig);
-                            filter = createScanningFilter(config);
-                            alreadyScannedClasses = config.getProperty("classes.list");
-                        }
-                    } catch (final IOException e) {
-                        log.debug(e.getMessage(), e);
+                        config.load(containerFilterConfig);
+                        filter = createScanningFilter(config);
+                        alreadyScannedClasses = config.getProperty("classes.list");
                     }
+                } catch (final IOException e) {
+                    log.debug(e.getMessage(), e);
                 }
 
                 AnnotationFinder optimizedFinder = null;

component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/service/MavenRepositoryDefaultResolver.java

@@ -52,6 +52,8 @@ public class MavenRepositoryDefaultResolver implements MavenRepositoryResolver {
 
     private PathHandler handler = new PathHandlerImpl();
 
+    private boolean hasFallback = Boolean.getBoolean("talend.component.manager.m2.fallback");
+
     @Override
     public Path discover() {
         return Stream
@@ -63,8 +65,12 @@ public Path discover() {
 
     @Override
     public Path fallback() {
-        log.debug("[fallback] default to user's default repository.");
-        return Paths.get(USER_HOME).resolve(M2_REPOSITORY);
+        log.debug("[fallback::{}] default to user's default repository.", hasFallback);
+        if (hasFallback) {
+            return Paths.get(USER_HOME).resolve(M2_REPOSITORY);
+        } else {
+            return Paths.get(USER_HOME);
+        }
     }
 
     public Path fromSystemProperties() {
@@ -119,30 +125,38 @@ private static String parseSettings(final Path settings) {
     }
 
     public Path fromMavenSettings() {
-        return Stream.of(
-                Optional.ofNullable(System.getProperty(TALEND_COMPONENT_MANAGER_M2_SETTINGS))
-                        .map(Paths::get)
-                        .orElse(null), //
-                Optional.ofNullable(System.getProperty("user.home")).map(it -> Paths.get(it, M2_SETTINGS)).orElse(null),
-                Optional.ofNullable(System.getenv(MAVEN_HOME)).map(it -> Paths.get(it, CONF_SETTINGS)).orElse(null),
-                Optional.ofNullable(System.getenv(M2_HOME)).map(it -> Paths.get(it, CONF_SETTINGS)).orElse(null))
-                .filter(Objects::nonNull)
-                .filter(Files::exists)
-                .map(MavenRepositoryDefaultResolver::parseSettings)
-                .filter(Objects::nonNull)
-                .map(handler::get)
-                .filter(Objects::nonNull)
-                .findFirst()
-                .orElse(null);
+        if (hasFallback) {
+            return Stream.of(
+                    Optional.ofNullable(System.getProperty(TALEND_COMPONENT_MANAGER_M2_SETTINGS))
+                            .map(Paths::get)
+                            .orElse(null), //
+                    Optional.ofNullable(System.getProperty("user.home"))
+                            .map(it -> Paths.get(it, M2_SETTINGS))
+                            .orElse(null),
+                    Optional.ofNullable(System.getenv(MAVEN_HOME)).map(it -> Paths.get(it, CONF_SETTINGS)).orElse(null),
+                    Optional.ofNullable(System.getenv(M2_HOME)).map(it -> Paths.get(it, CONF_SETTINGS)).orElse(null))
+                    .filter(Objects::nonNull)
+                    .filter(Files::exists)
+                    .map(MavenRepositoryDefaultResolver::parseSettings)
+                    .filter(Objects::nonNull)
+                    .map(handler::get)
+                    .filter(Objects::nonNull)
+                    .findFirst()
+                    .orElse(null);
+        } else {
+            return null;
+        }
     }
 
     public Path fromEnvironmentVariables() {
-        final String vm2 = System.getenv(M2_HOME);
-        log.debug("[fromEnvironmentVariables] M2_HOME={}", vm2);
-        if (vm2 != null) {
-            return handler.get(Paths.get(vm2, "repository").toString());
+        if (hasFallback) {
+            final String vm2 = System.getenv(M2_HOME);
+            log.debug("[fromEnvironmentVariables] M2_HOME={}", vm2);
+            if (vm2 != null) {
+                return handler.get(Paths.get(vm2, "repository").toString());
+            }
+            log.debug("[fromEnvironmentVariables] Could not get m2 from environment.");
         }
-        log.debug("[fromEnvironmentVariables] Could not get m2 from environment.");
         return null;
     }
 

component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/ComponentManagerTest.java

@@ -60,7 +60,9 @@
 import javax.management.ReflectionException;
 
 import org.apache.xbean.finder.util.Files;
+import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -98,6 +100,16 @@ private ComponentManager newManager() {
         return newManager(new File("target/test-dependencies"));
     }
 
+    @BeforeAll
+    static void setup() {
+        System.setProperty("talend.component.manager.m2.fallback", "true");
+    }
+
+    @AfterAll
+    static void teardown() {
+        System.clearProperty("talend.component.manager.m2.fallback");
+    }
+
     @Test
     void doubleClose() {
         final ComponentManager instance = ComponentManager.instance();
@@ -427,11 +439,10 @@ void extendFamilyInNestedRepo(@TempDir final File temporaryFolder) throws Except
                         .of(container.getLoader().getURLs())
                         .map(Files::toFile)
                         .map(File::getName)
-                        .sorted() // !! for asserts
+                        .sorted()
                         .toArray(String[]::new);
-                assertEquals(2, dependencies.length); // ignored transitive deps, enables the new root to control it
-                assertEquals("fatjar.jar", dependencies[0]); // transitive-1.0.0.jar is nested
-                assertEquals("main.jar", dependencies[1]); // main.jar containing fatjar.jar
+                assertEquals(1, dependencies.length); // ignored transitive deps, enables the new root to control it
+                assertEquals("main.jar", dependencies[0]); // transitive-1.0.0.jar is nested
             } finally {
                 if (!transitive.delete()) {
                     transitive.deleteOnExit();

component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/service/HttpClientFactoryImplTest.java

@@ -244,7 +244,7 @@ void requestWithXMLXXE() throws IOException {
             final byte[] bytes;
             String xmlContent = "<!DOCTYPE foo [ <!ENTITY xxe SYSTEM \"file:///etc/passwd\"> ]><foo>&xxe;</foo>";
             try (final BufferedReader in =
-                         new BufferedReader(new StringReader(xmlContent))) {
+                    new BufferedReader(new StringReader(xmlContent))) {
                 bytes = in.lines().collect(joining("\n")).getBytes(StandardCharsets.UTF_8);
             }
             httpExchange.sendResponseHeaders(HttpURLConnection.HTTP_OK, bytes.length);
@@ -273,7 +273,7 @@ void requestWithInvalidXML() throws IOException {
             final byte[] bytes;
             String xmlContent = "<!ENTITY xxe SYSTEM \"file:///etc/passwd\"><foo>invalid;";
             try (final BufferedReader in =
-                         new BufferedReader(new StringReader(xmlContent))) {
+                    new BufferedReader(new StringReader(xmlContent))) {
                 bytes = in.lines().collect(joining("\n")).getBytes(StandardCharsets.UTF_8);
             }
             httpExchange.sendResponseHeaders(HttpURLConnection.HTTP_OK, bytes.length);

component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/service/MavenRepositoryResolverTest.java

@@ -70,6 +70,7 @@ public Path get(final String path) {
 
     @BeforeAll
     static void setup() throws IOException {
+        System.setProperty("talend.component.manager.m2.fallback", "true");
         final Path repository = Paths.get(new File("target/test-classes").getAbsolutePath());
         Files.createDirectories(repository.resolve(M2_REPOSITORY));
     }

container/container-core/src/main/java/org/talend/sdk/component/classloader/ConfigurableClassLoader.java

@@ -111,8 +111,6 @@ public class ConfigurableClassLoader extends URLClassLoader {
     @Getter
     private final List<String> cacheableClasses;
 
-    private List<URL> nestedURLs = new ArrayList<>();
-
     public ConfigurableClassLoader(final String id, final URL[] urls, final ClassLoader parent,
             final Predicate<String> parentFilter, final Predicate<String> childFirstFilter,
             final String[] nestedDependencies, final String[] jvmPrefixes) {
@@ -157,7 +155,6 @@ private void loadNestedDependencies(final ClassLoader parent, final String[] nes
             if (url == null) {
                 throw new IllegalArgumentException("Didn't find " + resource + " in " + asList(nestedDependencies));
             }
-            nestedURLs.add(url);
             final Map<String, Resource> resources = new HashMap<>();
             final URLConnection urlConnection;
             final Manifest manifest;
@@ -461,15 +458,6 @@ public Enumeration<URL> findResources(final String name) throws IOException {
         return enumeration(aggregated);
     }
 
-    @Override
-    public URL[] getURLs() {
-        final List<URL> urls = new ArrayList<>(Arrays.asList(super.getURLs()));
-        if (!nestedURLs.isEmpty()) {
-            urls.addAll(nestedURLs);
-        }
-        return urls.toArray(new URL[0]);
-    }
-
     private boolean isNestedDependencyResource(final String name) {
         return name != null && name.startsWith(NESTED_MAVEN_REPOSITORY);
     }

container/container-core/src/main/java/org/talend/sdk/component/container/ContainerManager.java

@@ -29,6 +29,8 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -105,29 +107,21 @@
 
     private final Pattern versionWithMilestone = Pattern.compile("M\\d+$");
 
     public ContainerManager(final DependenciesResolutionConfiguration dependenciesResolutionConfiguration,
             final ClassLoaderConfiguration classLoaderConfiguration, final Consumer<Container> containerInitializer,
             final Level logInfoLevelMapping) {
         this.logInfoLevelMapping = logInfoLevelMapping;
         this.containerInitializer = containerInitializer;
         this.resolver = dependenciesResolutionConfiguration.getResolver();
+        this.rootRepositoryLocation = dependenciesResolutionConfiguration.getRootRepositoryLocation();
 
-        Path rootRepo = ofNullable(dependenciesResolutionConfiguration.getRootRepositoryLocation())
-                .filter(Files::exists)
-                .orElseGet(() -> PathFactory.get(System.getProperty("user.home")).resolve(".m2/repository"));
-        // if we've defaulted to user home m2 (fallback), we want to check if we're in running in a fatjar
-        if (PathFactory.get(System.getProperty("user.home")).resolve(".m2/repository").equals(rootRepo)) {
-            final URL nested = classLoaderConfiguration.getParent().getResource("MAVEN-INF/repository");
-            if (nested != null) {
-                rootRepo = PathFactory.get(nested.getFile().replace("file:", ""));
-            }
+        if (log.isDebugEnabled()) {
+            log.debug("Using root repository: " + this.rootRepositoryLocation.toAbsolutePath());
+            getSystemInformation();
         }
-        this.rootRepositoryLocation = rootRepo;
-        info("Using root repository: " + this.rootRepositoryLocation.toAbsolutePath());
-        final String pluginsLocation = System.getProperty("talend.component.manager.plugins.location",
-                "TALEND-INF/plugins.properties");
+
         final String nestedPluginMappingResource = ofNullable(classLoaderConfiguration.getNestedPluginMappingResource())
-                .orElse(pluginsLocation);
+                .orElse("TALEND-INF/plugins.properties");
         this.classLoaderConfiguration = new ClassLoaderConfiguration(
                 ofNullable(classLoaderConfiguration.getParent()).orElseGet(ContainerManager.class::getClassLoader),
                 ofNullable(classLoaderConfiguration.getClassesFilter()).orElseGet(() -> name -> true),
@@ -137,6 +131,12 @@
             try (final InputStream mappingStream =
                     classLoaderConfiguration.getParent().getResourceAsStream(nestedPluginMappingResource)) {
                 if (mappingStream != null) {
+                    if (log.isDebugEnabled()) {
+                        final URL plug = classLoaderConfiguration.getParent().getResource(nestedPluginMappingResource);
+                        if (plug != null) {
+                            log.debug("[sysinfo] plugins mapping " + plug.toString());
+                        }
+                    }
                     final Properties properties = new Properties() {
 
                         {
@@ -163,10 +163,13 @@
         this.jvmMarkers = Stream
                 .concat(Stream.concat(Stream.of(getJre()), getComponentModules()), getCustomJvmMarkers())
                 .toArray(String[]::new);
-        this.hasNestedRepository =
-                this.classLoaderConfiguration.isSupportsResourceDependencies() && this.classLoaderConfiguration
-                        .getParent()
-                        .getResource(ConfigurableClassLoader.NESTED_MAVEN_REPOSITORY) != null;
+        final URL nestedMvn = this.classLoaderConfiguration
+                .getParent()
+                .getResource(ConfigurableClassLoader.NESTED_MAVEN_REPOSITORY);
+        this.hasNestedRepository = this.classLoaderConfiguration.isSupportsResourceDependencies() && nestedMvn != null;
+        if (log.isDebugEnabled() && hasNestedRepository) {
+            log.debug("[sysinfo] nested maven repository: " + nestedMvn);
+        }
     }
 
     public File getRootRepositoryLocation() {
@@ -402,6 +405,22 @@
                 .orElseThrow(IllegalArgumentException::new);
     }
 
+    private void getSystemInformation() {
+        try {
+            final RuntimeMXBean rt = ManagementFactory.getRuntimeMXBean();
+            log.debug("[sysinfo] JVM arguments: " + rt.getInputArguments());
+            try {
+                log.debug("[sysinfo] Boot classpath: " + rt.getBootClassPath());
+            } catch (Exception e) {
+                // nop, will fail in some cases for boot classpath
+            }
+            log.debug("[sysinfo] Runtime classpath: " + rt.getClassPath());
+            log.debug("[sysinfo] Runtime arguments: " + System.getProperty("sun.java.command"));
+        } catch (Exception e) {
+            log.debug("Unable to get JVM information: " + e.getMessage(), e);
+        }
+    }
+
     @Override
     public void close() {
         lifecycle.closeIfNeeded(() -> {
@@ -485,8 +504,8 @@
                     ? nestedContainerMapping.getOrDefault(module, module)
                     : module;
             final Path resolved = resolve(moduleLocation);
-            info(String.format("Creating module %s (from %s, location=%s)", moduleLocation, module,
-                    resolved.toAbsolutePath()));
+            info("Creating module " + moduleLocation + " (from " + module
+                    + (Files.exists(resolved) ? ", location=" + resolved.toAbsolutePath().toString() : "") + ")");
             final Stream<Artifact> classpath = Stream
                     .concat(getBuiltInClasspath(moduleLocation),
                             additionalClasspath == null ? Stream.empty() : additionalClasspath.stream());