Required updates for compatibility

dlg_example_cmpts/__init__.py

@@ -8,7 +8,6 @@
     GenericGather,
     LengthCheck,
     MyBranch,
-    PickOne,
     String2JSON,
 )
 from .data import MyDataDROP
@@ -18,7 +17,6 @@
     "LengthCheck",
     "MyDataDROP",
     "FileGlob",
-    "PickOne",
     "String2JSON",
     "ExtractColumn",
     "AdvUrlRetrieve",

dlg_example_cmpts/apps.py

@@ -16,7 +16,7 @@
 import logging
 import os
 import pickle
-import urllib
+import requests
 from glob import glob
 
 import numpy as np
@@ -54,12 +54,12 @@
 #
 # @par EAGLE_START
 # @param category PythonApp
-# @param appclass Application class/dlg_example_cmpts.apps.MyBranch/String/ComponentParameter/readonly//False/False/Application class # noqa: E501
-# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time # noqa: E501
-# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used # noqa: E501
-# @param array Array//Object.Array/InputPort/readwrite//False/False/Port receiving the input array # noqa: E501
-# @param Y Y//float/OutputPort/readwrite//False/False/English Port carrying the mean value of the array if mean < 0.5 # noqa: E501
-# @param N N//float/OutputPort/readwrite//False/False/English Port carrying the mean value of the array if mean >= 0.5 # noqa: E501
+# @param dropclass dropclass/dlg_example_cmpts.apps.MyBranch/String/ComponentParameter/readonly//False/False/Application class # noqa: E501
+# @param execution_time execution_time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time # noqa: E501
+# @param num_cpus num_cpus/1/Integer/ComponentParameter/readonly//False/False/Number of cores used # noqa: E501
+# @param array array//Object.Array/InputPort/readwrite//False/False/Port receiving the input array # noqa: E501
+# @param yes yes//float/OutputPort/readwrite//False/False/English Port carrying the mean value of the array if mean < 0.5 # noqa: E501
+# @param no no//float/OutputPort/readwrite//False/False/English Port carrying the mean value of the array if mean >= 0.5 # noqa: E501
 # @par EAGLE_END
 
 
@@ -104,12 +104,12 @@ def condition(self):
 #
 # @par EAGLE_START
 # @param category PythonApp
-# @param appclass Application class/dlg_example_cmpts.apps.MyBranch/String/ComponentParameter/readonly//False/False/Application class # noqa: E501
-# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time # noqa: E501
-# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used # noqa: E501
-# @param array Array//Object.Array/InputPort/readwrite//False/False/Port receiving the input array # noqa: E501
-# @param Y Y//float/OutputPort/readwrite//False/False/English Port carrying the mean value of the array if mean < 0.5 # noqa: E501
-# @param N N//float/OutputPort/readwrite//False/False/English Port carrying the mean value of the array if mean >= 0.5 # noqa: E501
+# @param dropclass dropclass/dlg_example_cmpts.apps.MyBranch/String/ComponentParameter/readonly//False/False/Application class # noqa: E501
+# @param execution_time execution_time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time # noqa: E501
+# @param num_cpus num_cpus/1/Integer/ComponentParameter/readonly//False/False/Number of cores used # noqa: E501
+# @param array array//Object.Array/InputPort/readwrite//False/False/Port receiving the input array # noqa: E501
+# @param yes yes//float/OutputPort/readwrite//False/False/English Port carrying the mean value of the array if mean < 0.5 # noqa: E501
+# @param no no//float/OutputPort/readwrite//False/False/English Port carrying the mean value of the array if mean >= 0.5 # noqa: E501
 # @par EAGLE_END
 
 
@@ -176,9 +176,9 @@ def run(self):
 #
 # @par EAGLE_START
 # @param category PythonApp
-# @param appclass Application class/dlg_example_cmpts.apps.FileGlob/String/ComponentParameter/readonly//False/False/Application class # noqa: E501
-# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time # noqa: E501
-# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used # noqa: E501
+# @param dropclass dropclass/dlg_example_cmpts.apps.FileGlob/String/ComponentParameter/readonly//False/False/Application class # noqa: E501
+# @param execution_time execution_time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time # noqa: E501
+# @param num_cpus num_cpus/1/Integer/ComponentParameter/readonly//False/False/Number of cores used # noqa: E501
 # @param wildcard wildcard/"*"/String/InputPort/readwrite//False/False/Port receiving the search pattern # noqa: E501
 # @param filepath filepath/"."/String/InputPort/readwrite//False/False/Port receiving the path to search # noqa: E501
 # @param file_list file_list//Object.array/OutputPort/readwrite//False/False/Port carrying the resulting file_list # noqa: E501
@@ -218,65 +218,6 @@ def run(self):
         self.writeData()
 
 
-##
-# @brief PickOne
-# @details App that picks the first element of an input list, passes that
-# to all outputs, except the first one. The first output is used to pass
-# the remaining array on. This app is useful for a loop.
-#
-# @par EAGLE_START
-# @param category PythonApp
-# @param appclass Application Class/dlg_example_cmpts.apps.PickOne/String/ComponentParameter/readonly//False/False/Import path for application class # noqa: E501
-# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time # noqa: E501
-# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used # noqa: E501
-# @param rest_array rest_array//Object.array/InputPort/readwrite//False/FalseList of elements # noqa: E501
-# @param element element//Object.element/OutputPort/readwrite//False/False/Port carrying the first element of input array # noqa: E501
-# @param rest_array rest_array//Object.array/OutputPort/readwrite//False/False/Port carrying the rest array # noqa: E501
-# @par EAGLE_END
-
-
-class PickOne(BarrierAppDROP):
-    """
-    Simple app picking one element at a time. Good for Loops.
-    """
-
-    def readData(self):
-        input = self.inputs[0]
-        data = pickle.loads(droputils.allDropContents(input))
-
-        # make sure we always have a ndarray with at least 1dim.
-        if type(data) not in (list, tuple) and not isinstance(
-            data, (np.ndarray)
-        ):
-            raise TypeError
-        if isinstance(data, np.ndarray) and data.ndim == 0:
-            data = np.array([data])
-        else:
-            data = np.array(data)
-        value = data[0] if len(data) else None
-        rest = data[1:] if len(data) > 1 else np.array([])
-        return value, rest
-
-    def writeData(self, value, rest):
-        """
-        Prepare the data and write to all outputs
-        """
-        # write rest to array output
-        # and value to every other output
-        for output in self.outputs:
-            if output.name == "rest_array":
-                d = pickle.dumps(rest)
-                output.len = len(d)
-            else:
-                d = pickle.dumps(value)
-                output.len = len(d)
-            output.write(d)
-
-    def run(self):
-        value, rest = self.readData()
-        self.writeData(value, rest)
-
-
 ##
 # @brief ExtractColumn
 # @details App that extracts one column of an table-like ndarray. The array is
@@ -287,9 +228,9 @@ def run(self):
 #
 # @par EAGLE_START
 # @param category PythonApp
-# @param appclass Application Class/dlg_example_cmpts.apps.ExtractColumn/String/ComponentParameter/readonly//False/False/Import path for application class # noqa: E501
-# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time # noqa: E501
-# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used # noqa: E501
+# @param dropclass dropclass/dlg_example_cmpts.apps.ExtractColumn/String/ComponentParameter/readonly//False/False/Import path for application class # noqa: E501
+# @param execution_time execution_time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time # noqa: E501
+# @param num_cpus num_cpus/1/Integer/ComponentParameter/readonly//False/False/Number of cores used # noqa: E501
 # @param index index/0/Integer/ApplicationArgument/readwrite//False/False/0-base index of column to extract # noqa: E501
 # @param table_array table_array//array/InputPort/readwrite//False/False/List of elements # noqa: E501
 # @param column column//Object.1Darray/OutputPort/readwrite//False/False/Port carrying the first element of input array # noqa: E501
@@ -336,12 +277,12 @@ def run(self):
 # use placeholders to construct the final URL.
 # @par EAGLE_START
 # @param category PythonApp
-# @param appclass Application Class/dlg_example_cmpts.apps.AdvUrlRetrieve/String/ComponentParameter/readonly//False/False/Application class # noqa: E501
-# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time # noqa: E501
-# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used # noqa: E501
-# @param urlTempl URL Template/"https://eagle.icrar.org"/String/ApplicationArgument/readwrite//False/False/The URL to retrieve # noqa: E501
-# @param urlPart URL Part//String/InputPort/readwrite//False/False/The port carrying the content read from the URL. # noqa: E501
-# @param content Content//String/OutputPort/readwrite//False/False/The port carrying the content read from the URL. # noqa: E501
+# @param dropclass droplass/dlg_example_cmpts.apps.AdvUrlRetrieve/String/ComponentParameter/readonly//False/False/Application class # noqa: E501
+# @param execution_time execution_time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time # noqa: E501
+# @param num_cpus num_cpus/1/Integer/ComponentParameter/readonly//False/False/Number of cores used # noqa: E501
+# @param urlTempl urlTempl/"https://eagle.icrar.org"/String/ApplicationArgument/readwrite//False/False/The URL to retrieve # noqa: E501
+# @param urlPart urlPart//String/InputPort/readwrite//False/False/The port carrying the content read from the URL. # noqa: E501
+# @param content content//String/OutputPort/readwrite//False/False/The port carrying the content read from the URL. # noqa: E501
 # @par EAGLE_END
 class AdvUrlRetrieve(BarrierAppDROP):
     """
@@ -382,7 +323,7 @@ def constructUrl(self):
     def readData(self):
         # for this app we are expecting URL fractions on the inputs
         urlParts = {}
-        named_inputs = self._generateNamedInputs()
+        named_inputs = self._generateNamedPorts("inputs")
         idict = named_inputs
         if len(named_inputs) > 0:
             idict = named_inputs
@@ -392,7 +333,7 @@ def readData(self):
         # elif isinstance(named_inputs, dict):
         #     idict = self.inputs
         logger.debug("Inputs identified: %s", idict)
-        for (name, input) in idict.items():
+        for name, input in idict.items():
             part = pickle.loads(droputils.allDropContents(input))
             # make sure the placeholders are strings
             logger.info(f"URL part: {name}:{part}")
@@ -403,11 +344,11 @@ def readData(self):
         self.url = self.constructUrl()
         logger.info("Reading from %s", self.url)
         try:
-            u = urllib.request.urlopen(self.url)
+            u = requests.get(self.url)
         except Exception as e:
             raise e
         # finally read the content of the URL
-        self.content = u.read()
+        self.content = u.content
 
     def writeData(self):
         """
@@ -443,14 +384,13 @@ def run(self):
 # JSON value is taken from there.
 # @par EAGLE_START
 # @param category PythonApp
-# @param appclass Application Class/dlg_example_cmpts.apps.String2JSON/String/ComponentParameter/readonly//False/False/Application class # noqa: E501
-# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time # noqa: E501
-# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used # noqa: E501
+# @param dropclass dropclass/dlg_example_cmpts.apps.String2JSON/String/ComponentParameter/readonly//False/False/Application class # noqa: E501
+# @param execution_time execution_time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time # noqa: E501
+# @param num_cpus num_cpus/1/Integer/ComponentParameter/readonly//False/False/Number of cores used # noqa: E501
 # @param string string//String/InputPort/readwrite//False/False/String to be converted # noqa: E501
 # @param element element//Object.json/OutputPort/readwrite//False/False/Port carrying the JSON structure # noqa: E501
 # @par EAGLE_END
 class String2JSON(BarrierAppDROP):
-
     string = dlg_string_param("string", None)
 
     def readData(self):
@@ -496,15 +436,14 @@ def run(self):
 #
 # @par EAGLE_START
 # @param category PythonApp
-# @param appclass Application Class/dlg_example_cmpts.apps.GenericGather/String/ComponentParameter/readonly//False/False/Import path for application class # noqa: E501
-# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time # noqa: E501
-# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used # noqa: E501
+# @param dropclass dropclass/dlg_example_cmpts.apps.GenericGather/String/ComponentParameter/readonly//False/False/Import path for application class # noqa: E501
+# @param execution_time execution_time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time # noqa: E501
+# @param num_cpus num_cpus/1/Integer/ComponentParameter/readonly//False/False/Number of cores used # noqa: E501
 # @param input input//Object/InputPort/readwrite//False/False/0-base placeholder port for inputs # noqa: E501
 # @param output output//Object/OutputPort/readwrite//False/False/Placeholder port for outputs # noqa: E501
 # @par EAGLE_END
 class GenericGather(BarrierAppDROP):
     def readWriteData(self):
-
         inputs = self.inputs
         outputs = self.outputs
         total_len = 0

dlg_example_cmpts/data.py

@@ -1,5 +1,6 @@
-import logging
 import io
+import logging
+
 from dlg.meta import dlg_string_param
 
 try:

tests/test_cmpts.py

@@ -1,33 +1,36 @@
-import pytest, unittest
+"""
+Test module for example components.
+"""
+from glob import glob
 import os
 import pickle
 import urllib
 import http
+import logging
+import json
+from time import sleep
+import pytest, unittest
+
+import numpy as np
 
-from glob import glob
 from dlg import droputils
+from dlg.apps.simple import RandomArrayApp
 
 from dlg_example_cmpts import (
     MyBranch,
     LengthCheck,
     MyDataDROP,
     FileGlob,
-    PickOne,
     String2JSON,
     ExtractColumn,
     AdvUrlRetrieve,
     GenericGather,
 )
-from dlg.apps.simple import RandomArrayApp
 
 try:
     from dlg.data.drops import InMemoryDROP, NullDROP
 except ImportError:
     from dlg.drop import InMemoryDROP, NullDROP
-import logging
-import json
-import numpy as np
-from time import sleep
 
 logger = logging.Logger(__name__)
 
@@ -241,59 +244,6 @@ def test_FileGlob_class_nomatch(self):
         res = pickle.loads(droputils.allDropContents(m))
         self.assertEqual(fileList, res)
 
-    def test_PickOne_class(self):
-        """
-        Testing the PickOne app
-        """
-        i = NullDROP("i", "i")  # just to be able to start the execution
-        l = RandomArrayApp("l", "l")
-        l.integer = True
-        l.high = 100
-        l.size = 10
-        a = InMemoryDROP("a", "a", type="array", nm="start_array")
-        p = PickOne("p", "p")
-        r = InMemoryDROP("r", "r", type="array", nm="rest_array")
-        o = InMemoryDROP("o", "o", type="scalar", nm="value")
-
-        i.addConsumer(l)
-        a.addProducer(l)
-        p.addInput(a)
-        r.addProducer(p)  # this should contain the rest elements
-        o.addProducer(p)  # this should contain the first element
-        with droputils.DROPWaiterCtx(self, o, timeout=10):
-            i.setCompleted()
-        in_array = pickle.loads(droputils.allDropContents(a))
-        first = pickle.loads(droputils.allDropContents(o))
-        rest_array = pickle.loads(droputils.allDropContents(r))
-        self.assertEqual(in_array[0], first)
-        self.assertEqual(all(in_array[1:]), all(rest_array))
-
-    def test_PickOne_wrong_input_type(self):
-        """
-        Testing the PickOne with wrong input type
-        """
-        a = InMemoryDROP("a", "a", type="array", nm="start_array")
-        a.write(pickle.dumps(1))  # this is scalar not array
-        p = PickOne("p", "p")
-        a.addConsumer(p)
-        with self.assertRaises(TypeError):
-            a.setCompleted()
-            p.readData()
-
-    def test_PickOne_0dim(self):
-        """
-        Testing the PickOne with 0dim input
-        """
-        a = InMemoryDROP("a", "a")
-        a.write(pickle.dumps(np.array(1)))  # this is 0dim not array
-        a.name = "start_array"
-        p = PickOne("p", "p")
-
-        p.addInput(a)
-        with droputils.DROPWaiterCtx(self, p, timeout=10):
-            a.setCompleted()
-        self.assertRaises(TypeError)
-
     def test_String2JSON(self):
         """
         Testing String2JSON with correct input