Merge pull request #2398 from RolefH/master

Allow main composites w/o SPL namespace to compile, issue #2285

com.ibm.streamsx.topology/opt/python/packages/streamsx/scripts/runner.py

@@ -127,7 +127,8 @@ def _get_topology_app(cmd_args):
     return app
 
 def _get_spl_app(cmd_args):
-    topo = op.main_composite(kind=cmd_args.main_composite,
+    # the private function permits main_composite w/o namespace 
+    topo = op._main_composite(kind=cmd_args.main_composite,
         toolkits=cmd_args.toolkits)[0]
     if cmd_args.create_bundle and 'STREAMS_INSTALL' in os.environ:
         # Mimic what the build service does by indexing

com.ibm.streamsx.topology/opt/python/packages/streamsx/scripts/sc.py

@@ -10,7 +10,7 @@
 import shutil
 import streamsx.rest
 import pkg_resources
-from streamsx.spl.op import main_composite
+from streamsx.spl.op import _main_composite
 from streamsx.spl.toolkit import add_toolkit
 from streamsx.topology.context import submit, ConfigParams
 from streamsx.build import BuildService
@@ -328,7 +328,8 @@ def _is_likely_toolkit(tkdir):
             return True
 
 def _create_topo(cmd_args):
-    topo,invoke = main_composite(kind=cmd_args.main_composite)
+    # the private function permits main composites w/o a namespace
+    topo,invoke = _main_composite(kind=cmd_args.main_composite)
     return topo
 
 def _parse_args(args):

com.ibm.streamsx.topology/opt/python/packages/streamsx/spl/op.py

@@ -437,6 +437,29 @@ def spl_json(self):
     def __str__(self):
         return str(self._value)
 
+def _main_composite(kind, toolkits=None, name=None):
+    """This private function is used by the scripts to support compilation of SPL Main composites w/o a namespace.
+    """ 
+    if '::' in kind:
+        ns, topo_name = kind.rsplit('::', 1)
+        ns += '._spl'
+    elif not name:
+        ns = '_spl'
+        topo_name = kind
+    else:
+        # wrapping a composite w/o namespace with a namespace qualified name ('_spl')
+        # would fail to compile with Streamsx Compiler
+        raise ValueError('Main composite requires a namespace qualified name: ' + str(kind))
+    if name:
+        topo_name = name
+    topo = streamsx.topology.topology.Topology(name=topo_name, namespace=ns)
+    if toolkits:
+        for tk_path in toolkits:
+            streamsx.spl.toolkit.add_toolkit(topo, tk_path)
+    if not name:
+        topo.graph._main_composite = kind
+    return topo, Invoke(topo, kind, name=name)
+
 def main_composite(kind, toolkits=None, name=None):
     """Wrap a main composite invocation as a `Topology`.
   
@@ -457,7 +480,7 @@ def main_composite(kind, toolkits=None, name=None):
     composite invocation is invoked within a generated main composite.
 
     Args:
-        kind(str): Kind of the main composite operator invocation.
+        kind(str): Kind of the main composite operator invocation. Must be a namespace qualified name.
         toolkits(list[str]): Optional list of toolkits the main composite depends on.
         name(str): Invocation name for the main composite.
 
@@ -470,16 +493,14 @@ def main_composite(kind, toolkits=None, name=None):
     .. versionadded: 1.11
     """
     if '::' in kind:
-        ns, topo_name = kind.rsplit('::', 1)
-        ns += '._spl'
+        pass
     else:
+        # A composite w/o namespace fails to compile by sc when the given main composite 
+        # is wrapped by another composite. This happens when:
+        #   a) A name is used
+        #   b) The returned topology is touched by adding operators (tester, another independent flow)
+        # 
+        # a) could be checked here, b) cannot be checked here - we had to mark the 
+        # topology somehow immutable, so that additions raise an Exception (TODO).
         raise ValueError('Main composite requires a namespace qualified name: ' + str(kind))
-    if name:
-        topo_name = name
-    topo = streamsx.topology.topology.Topology(name=topo_name, namespace=ns)
-    if toolkits:
-        for tk_path in toolkits:
-            streamsx.spl.toolkit.add_toolkit(topo, tk_path)
-    if not name:
-        topo.graph._main_composite = kind
-    return topo, Invoke(topo, kind, name=name)
+    return _main_composite(kind, toolkits, name)

java/src/com/ibm/streamsx/topology/generator/spl/SPLGenerator.java

@@ -303,6 +303,7 @@ else if(compStarts.get(i).equals(BVirtualMarker.LOW_LATENCY.kind())){
      * @param startsEndsAndOperators
      * @param opDefinition
      */
+    @SuppressWarnings("unused")
     private void fixCompositeInputNaming(JsonObject graph, List<List<JsonObject>> startsEndsAndOperators,
             JsonObject opDefinition) {   
         // For each start
@@ -349,6 +350,7 @@ private void fixCompositeInputNaming(JsonObject graph, List<List<JsonObject>> st
      * @param startsEndsAndOperators
      * @param opDefinition
      */
+    @SuppressWarnings("unused")
     private void fixCompositeOutputNaming(JsonObject graph, List<List<JsonObject>> startsEndsAndOperators,
             JsonObject opDefinition, JsonObject opInvocation) {   
         // We iterate like this because we need the index into the operatorDefinition's inputNames list.
@@ -980,6 +982,8 @@ SubmissionTimeValue stvHelper() {
         return stvHelper;
     }
 
+    private static final int NAME_LEN = 80;
+
     /**
      * Takes a name String that might have characters which are incompatible in
      * an SPL stream name (which just supports ASCII) and returns a valid SPL
@@ -1004,12 +1008,11 @@ SubmissionTimeValue stvHelper() {
      * @return A string which can be a valid SPL stream name. If name is valid
      * as an SPL identifier and less than 80 chars then it is returned (same reference).
      */
-    private static final int NAME_LEN = 80;
     public static String getSPLCompatibleName(String name) {
 
-        if (name.length() <= NAME_LEN && name.matches("^[a-zA-Z_][a-zA-Z0-9_]*$"))
+        if (name.length() <= NAME_LEN && name.matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) {
             return name;
-
+        }
         final byte[] original = name.getBytes(StandardCharsets.UTF_8);
         return "__spl_" + md5Name(original);
     }

java/src/com/ibm/streamsx/topology/internal/context/remote/DeployKeys.java

@@ -123,7 +123,7 @@ static JsonObject copyJobConfigOverlays(JsonObject deploy) {
     }
 
     /**
-     * Save a JobConfig overlay file.
+     * Save a JobConfig overlay file, adds a submissionResult to the submission JsonObject.
      */
     static File createJobConfigOverlayFile(JsonObject submission, JsonObject deploy, File dir) throws IOException {
 
@@ -134,7 +134,7 @@ static File createJobConfigOverlayFile(JsonObject submission, JsonObject deploy,
             String name = splAppName(graph);
 
             final JsonObject submissionResult = GsonUtilities.objectCreate(submission, RemoteContext.SUBMISSION_RESULTS);
-
+            // adds the "jobConfigPath" as side-effect to the submissionResult
             return createJobConfigOverlayFile(dir, copyJobConfigOverlays(deploy),
                     namespace, name, submissionResult);
         }
@@ -143,13 +143,18 @@ static File createJobConfigOverlayFile(JsonObject submission, JsonObject deploy,
 
     static File createJobConfigOverlayFile(File dir, JsonObject jco, String namespace, String name,
             JsonObject result) throws IOException {
-
-        String sabBaseName = namespace == null ? name : namespace + "." + name;
+        final boolean haveNameSpace = namespace != null && !namespace.isEmpty();
+        String sabBaseName = haveNameSpace? namespace + "." + name: name;
         File jcf = new File(dir, sabBaseName + "_JobConfig.json");
 
-        jco.addProperty("comment",
-                String.format("Job Config Overlays for %s::%s - generated %s", namespace, name, new Date()));
-
+        if (haveNameSpace) {
+            jco.addProperty("comment",
+                    String.format("Job Config Overlays for %s::%s - generated %s", namespace, name, new Date()));
+        }
+        else {
+            jco.addProperty("comment",
+                    String.format("Job Config Overlays for %s - generated %s", name, new Date()));
+        }
         String jcos_str = gson().toJson(jco);
 
         Files.write(jcf.toPath(), jcos_str.getBytes(StandardCharsets.UTF_8));

java/src/com/ibm/streamsx/topology/internal/context/remote/ToolkitRemoteContext.java

@@ -184,14 +184,25 @@ private void generateSPL(File toolkitRoot, JsonObject jsonGraph)
         createNamespaceFile(toolkitRoot, jsonGraph, "spl", generator.generateSPL(jsonGraph));      
     }
 
+    /**
+     * Create a file in the SPL Namespace directory toolkitRoot/splnamespace/appName.suffix
+     * @param toolkitRoot The root directory of the toolkit
+     * @param json The JSON graph, from which the SPL namespace and the name of the file is derived 
+     * @param suffix the file suffix, separated with a dot (file extension)
+     * @param content The content of the file
+     * @throws IOException
+     */
     private void createNamespaceFile(File toolkitRoot, JsonObject json, String suffix, String content)
             throws IOException {
 
         String namespace = splAppNamespace(json);
         String name = splAppName(json);
-
-        Path f = Paths.get(toolkitRoot.getAbsolutePath(), namespace, name + "." + suffix);
-
+        Path f;
+        if (namespace != null && !namespace.isEmpty()) {
+            f = Paths.get(toolkitRoot.getAbsolutePath(), namespace, name + "." + suffix);
+        } else {
+            f = Paths.get(toolkitRoot.getAbsolutePath(), name + "." + suffix);
+        }
         try (PrintWriter splFile = new PrintWriter(f.toFile(), UTF_8.name())) {
             splFile.print(content);
             splFile.flush();
@@ -200,14 +211,14 @@ private void createNamespaceFile(File toolkitRoot, JsonObject json, String suffi
 
     public static void makeDirectoryStructure(File toolkitRoot, String namespace)
             throws Exception {
-
-        File tkNamespace = new File(toolkitRoot, namespace);
+        final boolean haveNamespace = namespace != null && !namespace.isEmpty();
+        File tkNamespace = haveNamespace? new File(toolkitRoot, namespace): toolkitRoot;
         File tkImplLib = new File(toolkitRoot, Paths.get("impl", "lib").toString());
         File tkEtc = new File(toolkitRoot, "etc");
         File tkOptDepends = new File(toolkitRoot, DEP_JAR_LOC);
 
-        tkImplLib.mkdirs();
         tkNamespace.mkdirs();
+        tkImplLib.mkdirs();
         tkEtc.mkdir();
         tkOptDepends.mkdirs();
     }
@@ -220,9 +231,15 @@ private void addToolkitInfo(File toolkitRoot, JsonObject jsonGraph) throws JAXBE
         File infoFile = new File(toolkitRoot, "info.xml");
 
         ToolkitInfoModelType info = new ToolkitInfoModelType();
-
+        final String namespace = GraphKeys.splAppNamespace(jsonGraph);
+        final boolean haveNamespace = namespace != null && !namespace.isEmpty();
         info.setIdentity(new IdentityType());
-        info.getIdentity().setName(GraphKeys.splAppNamespace(jsonGraph) + "." + GraphKeys.splAppName(jsonGraph));
+        // toolkit name in info.xml
+        if (haveNamespace) {
+            info.getIdentity().setName(namespace + "." + GraphKeys.splAppName(jsonGraph));
+        } else {
+            info.getIdentity().setName(GraphKeys.splAppName(jsonGraph));
+        }
         info.getIdentity().setDescription(new DescriptionType());
         info.getIdentity().setVersion("1.0.0." + System.currentTimeMillis());
         info.getIdentity().setRequiredProductVersion(jstring(object(jsonGraph, "config"), CFG_STREAMS_VERSION));

java/src/com/ibm/streamsx/topology/internal/context/remote/ZippedToolkitRemoteContext.java

@@ -107,7 +107,11 @@ private static Path pack(final File appTkRoot, JsonObject submission, String tkN
         if (mainCompositeKind == null) {
             String namespace = splAppNamespace(graph);
             String name = splAppName(graph);
-            mainCompositeKind = namespace + "::" + name;
+            if (namespace != null && !namespace.isEmpty()) {
+                mainCompositeKind = namespace + "::" + name;
+            } else {
+                mainCompositeKind = name;
+            }
         }
 
         Path zipFilePath = Files.createTempFile("code_archive", ".zip");

java/src/com/ibm/streamsx/topology/internal/context/streams/BundleStreamsContext.java

@@ -74,29 +74,36 @@ protected Future<File> action(AppEntity entity) throws Exception {
         JsonObject deploy = deploy(submission);
 
         File appDir;
-        if (mainCompositeKind == null) {
-            if (deploy.has(APP_DIR))
-                deploy.add(TOOLKIT_DIR, deploy.get(APP_DIR));
-
-            appDir = super.action(entity).get();
-        } else {
-            // Single main composite, no need for a toolkit.
+        if (mainCompositeKind != null) {
+            // Single main composite, no need to generate a toolkit before the SPL compile step.
             String namespace;
             String name;
             int sep = mainCompositeKind.indexOf("::");
             if (sep != -1) {
                 namespace = mainCompositeKind.substring(0, sep);
                 name = mainCompositeKind.substring(sep+2);
             } else {
-                namespace = null;
+                // main composite w/o namespace.
+                // Add an empty "splNamespace" for the compile step to the graph.
+                // If we did not do this, or added 'null', the "namespace" from the graph
+                // would be used for the compiler. This is the namespace of the _topology_, to which the
+                // main composite has been added. The namespace of the topology is always the SPL-namespace
+                // of the main composite extended by '._spl' or '_spl'. So, we tried to compile '_spl::Main'
+                // instead of 'Main' --> compiler error '_spl::Main not found'.
+                namespace = "";
                 name = mainCompositeKind;
             }
             graph.addProperty(GraphKeys.SPL_NAMESPACE, namespace);
             graph.addProperty(GraphKeys.SPL_NAME, name);
 
             appDir = Files.createTempDirectory("tk").toFile();
-
             ToolkitRemoteContext.setupJobConfigOverlays(deploy, graph);
+        } else {
+            // generate a an SPL toolkit
+            if (deploy.has(APP_DIR))
+                deploy.add(TOOLKIT_DIR, deploy.get(APP_DIR));
+
+            appDir = super.action(entity).get();
         }
     	Future<File> bundle = doSPLCompile(appDir, submission);
 
@@ -118,7 +125,6 @@ protected Future<File> action(AppEntity entity) throws Exception {
 
 
     private Future<File> doSPLCompile(File appDir, JsonObject submission) throws Exception {
-
     	JsonObject deploy = deploy(submission);
     	JsonObject graph = GraphKeys.graph(submission);
 
@@ -142,8 +148,9 @@ private Future<File> doSPLCompile(File appDir, JsonObject submission) throws Exc
 
         sc.invoke();
 
+        final boolean haveNamespace = namespace != null && !namespace.isEmpty();
         File outputDir = new File(appDir, "output");
-        String bundleName = namespace + "." + name + ".sab";
+        String bundleName = haveNamespace? namespace + "." + name + ".sab": name + ".sab";
         File bundle = new File(outputDir, bundleName);
 
         File localBundle = new File(bundle.getName());

java/src/com/ibm/streamsx/topology/internal/streams/InvokeSc.java

@@ -39,7 +39,7 @@ public InvokeSc(JsonObject deploy, boolean standalone, String namespace, String
             File applicationDir) throws URISyntaxException, IOException {
         super();
 
-        this.namespace = namespace;
+        this.namespace = namespace == null? "": namespace;
         this.mainComposite = mainComposite;
         this.applicationDir = applicationDir;
 
@@ -103,7 +103,7 @@ public void invoke() throws Exception, InterruptedException {
 
         List<String> commands = new ArrayList<>();
 
-        String mainCompositeName = namespace + "::" + mainComposite;
+        String mainCompositeName = namespace.isEmpty()? mainComposite: namespace + "::" + mainComposite;
 
         commands.add(sc.getAbsolutePath());
         commands.add("--rebuild-toolkits");

test/python/scripts/sc_test_files/apps/readme.txt

@@ -0,0 +1,11 @@
+Name SPL files or SPLMM files that do not live in a namespace directory
+
+- test_no_ns.spl.tmpl     (pure SPL files)
+- test_no_ns.splmm.tmpl   (SPLMM files)
+
+Artifacts in the namespace directory tmp.samplemain must be named
+
+- test.spl.tmpl     (pure SPL files)
+- test.splmm.tmpl   (SPLMM files)
+
+Otherwise the scripts create_test_sc_files.sh and delete_test_sc_files.sh won't work properly.