Feature improve read speed mk2 rusu1 adjustments (#271)

* Update read to Improve Performance and Readability

.github/workflows/release.yml

@@ -18,7 +18,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
         with:
-          tag_name: pydv-3.1.1
-          release_name: PyDV 3.1.1
+          tag_name: pydv-3.1.2
+          release_name: PyDV 3.1.2
           draft: false
           prerelease: false

pydv/docs/conf.py

@@ -60,7 +60,7 @@
 # The short X.Y version.
 version = u'3.1'
 # The full version, including alpha/beta/rc tags.
-release = u'3.1.1'
+release = u'3.1.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

pydv/docs/release_notes.rst

@@ -3,6 +3,11 @@
 Release Notes
 =============
 
+3.1.2
+-----
+* Improved `pydvpy.read()` performance.
+
+
 3.1.1
 -----
 ``@`` notation in curve indexing fixed for mathematical operations.

pydv/pdv.py

@@ -6678,7 +6678,7 @@ def tickFormat(self, axis, logscale, ticks, tickformat):
 
     def console_run(self):
         while True:
-            self.cmdloop('\n\tPython Data Visualizer 3.1.1  -  05.23.2022\n\tType "help" for more information.\n\n')
+            self.cmdloop('\n\tPython Data Visualizer 3.1.2  -  08.11.2022\n\tType "help" for more information.\n\n')
             print('\n   Starting Python Console...\n   Ctrl-D to return to PyDV\n')
             console = code.InteractiveConsole(locals())
             console.interact()

pydv/pdvplot.py

@@ -114,7 +114,7 @@ def __init__(self, pydvcmd):
         here = path.abspath(path.dirname(__file__))
 
         # Setup Application
-        self.setWindowTitle('Python Data Visualizer 3.1.1')
+        self.setWindowTitle('Python Data Visualizer 3.1.2')
         self.setWindowIcon(QIcon(path.join(here, 'img/app_icon3.png')))
         self._pydvcmd = pydvcmd
 
@@ -519,7 +519,7 @@ def __aboutQt(self):
 
     def __aboutPyDV(self):
         QMessageBox.about(self, self.tr('About PyDV'), self.tr('<h2>About PyDV</h2>'
-                                                               '<p style="font-family:courier; font-size:40%;">version 3.1.1</p>'
+                                                               '<p style="font-family:courier; font-size:40%;">version 3.1.2</p>'
                                                                '<p style="font-family:verdana;"><a href="https://pydv.readthedocs.io/en/latest/">PyDV</a> is a 1D graphics tool, heavily based on the ULTRA plotting tool.</p>'
                                                                '<p style="font-family:courier; font-size:-1;">Copyright &copy; 2011-2022, Lawrence Livermore National Security, LLC.</p>'
                                                                '<p style="font-family:veranda; font-size:80%;">Written by: Edward Rusu, Kevin Griffin, Mason Kwiat, and Douglas S. Miller</p>'

pydv/pydvpy.py

@@ -414,16 +414,16 @@ def savecsv(fname, curvelist, verbose=False):
             f.close()
 
 
-def read(fname, gnu=False, xcol=0, verbose=False, pattern=None, matches=None):
+def read(file_name, gnu=False, xcol=0, verbose=False, pattern=None, matches=None):
     """
     Read the file and add parsed curves to a curvelist
 
     >>> curves = pydvif.read('testData.txt')
 
     >>> curves = pydvif.read('testData.txt', False, 0, False, '*_name', 20)
 
-    :param fname: ULTRA filename
-    :type fname: str
+    :param file_name: ULTRA filename
+    :type file_name: str
     :param gnu: optional, flag to determine if the file is a column oriented (.gnu) file.
     :type gnu: bool
     :param xcol: optional, x-column number for column oriented (.gnu) files
@@ -437,146 +437,98 @@ def read(fname, gnu=False, xcol=0, verbose=False, pattern=None, matches=None):
     :returns: list -- the list of curves from the file matching pattern, if specified
     
     """
-    curvelist = list()
+    def bundle_curve(_curve, build_x, build_y):
+        if len(build_x) != len(build_y):
+            build_y.append(build_y[-1])
+
+            _curve.x = np.array(build_x, dtype=float).repeat(2)[1:]
+            _curve.y = np.array(build_y, dtype=float).repeat(2)[:-1]
+        else:
+            _curve.x = np.array(build_x, dtype=float)
+            _curve.y = np.array(build_y, dtype=float)
+
+        return _curve
+
+    curve_list = list()
     regex = None
-    matchcount = 0
 
-    if pattern is not None:
+    if pattern:
         regex = re.compile(r"%s" % pattern)
-
+    file_name = os.path.expanduser(file_name)
+    _, ext = os.path.splitext(file_name)
     try:
-        if fname.strip()[0] == '~':
-            fname = os.getenv('HOME') + fname.strip()[1:]
-
-        if gnu or fname[-4:] == '.gnu':
-            return __loadcolumns(fname, xcol)
+        if gnu or ext == '.gnu':
+            return __loadcolumns(file_name, xcol)
 
         if pdbLoaded:
             try:
-                fpdb = pdb.open(fname, 'r')
-                return __loadpdb(fname, fpdb)
+                fpdb = pdb.open(file_name, 'r')
+                return __loadpdb(file_name, fpdb)
             except:
                 pass
 
-        f = open(fname, 'r')
-        current = None # curve.Curve(fname, "Unlabeled")
-        first = 1
-        buildlistx = list()
-        buildlisty = list()
-
-        for line in f:
-            if line.strip()[:2] == '##':
-                continue
-
-            if line.strip()[:1] == '#':  # check for start of new curve
-                if matches is not None:
-                    # print "matches = %d" % matches
-                    if matchcount >= matches:
-                        break
-
-                if first != 1:
-                    if len(buildlistx) != 0:
-                        if current.drawstyle != 'default':
-                            current.drawstyle = 'default'
-                            buildlisty.append(buildlisty[-1])
-                            histx = list()
-                            histx.append(buildlistx[0])
-                            histy = list()
-                            histy.append(buildlisty[0])
-
-                            i = 1
-                            while i < len(buildlisty):
-                                # insert new x,y
-                                histx.append(buildlistx[i])
-                                histy.append(buildlisty[i - 1])
-
-                                # insert old x,y
-                                histx.append(buildlistx[i])
-                                histy.append(buildlisty[i])
-
-                                i += 1
-
-                            current.x = np.array(histx)
-                            current.y = np.array(histy)
-                        else:
-                            current.x = np.array(buildlistx)
-                            current.y = np.array(buildlisty)
-
-                        curvelist.append(current)
-                        buildlistx = list()
-                        buildlisty = list()
+        match_count = 0
+        build_list_x = list()
+        build_list_y = list()
+        current = None
+        new_curve = True
+        potential_curve_name = ""
+        with open(file_name, 'r') as f:
+            for line in f:
+                split_line = re.split(r'[ _]+', str.strip(line))
+                if not split_line or not split_line[0]:
+                    continue
+                elif split_line[0] in {'##', 'end', 'End', 'END'}:
+                    continue
+                elif split_line[0] == '#':
+                    # We may have just finished buiding a curve, so we need to
+                    # add it to the list of curves.
+                    # If this is the first curve, then current will be None, so
+                    # we won't add anything.
+                    # If there is a sequence of lines that start with # before
+                    # getting to the actual data, then the new_curve flag will
+                    # keep us from adding all those comments as curves.
+                    if current and not new_curve: 
+                        curve_list.append(bundle_curve(current, build_list_x, build_list_y))
+                        build_list_x = list()
+                        build_list_y = list()
+
+                    # Beging setup of new curve
+                    new_curve = True
+                    potential_curve_name = ' '.join(split_line[1:])
                 else:
-                    first = 0
+                    if new_curve:
+                        curve_name = potential_curve_name
+                        new_curve = False
+                        if regex:
+                            if regex.search(curve_name):
+                                match_count += 1
+                                current = curve.Curve(file_name, curve_name)
+                            else:
+                                current = None
+                        else:
+                            current = curve.Curve(file_name, curve_name)
 
-                curvename = line.strip()[1:]
-                curvename = curvename.strip()
+                    build_list_x += split_line[::2]
+                    build_list_y += split_line[1::2]
 
-                if regex is not None:
-                    if regex.search(curvename) is not None:
-                        matchcount += 1
-                        current = curve.Curve(fname, curvename)
-                    else:
-                        current = None
-                else:
-                    current = curve.Curve(fname, curvename)
-            elif line.strip().lower() == 'end':
-                pass
-            else:
-                if current is not None:
-                    vals = line.split()
-                    dim = 'x'
-                    for i in range(len(vals)):  # parse x and y values
-                        if dim == 'x':
-                            if (i+1) < len(vals):   # Fixes PYDV-84
-                                buildlistx.append(float(vals[i]))
-                                buildlisty.append(float(vals[i+1]))
-                            else:   # treat as piecewise constant
-                                buildlistx.append(float(vals[i]))
-                                current.drawstyle = 'steps-post'
-                            dim = 'skip'
-                        elif dim == 'skip':
-                            dim = 'x'
-
-        if current is not None:
-            if current.drawstyle != 'default':
-                current.drawstyle = 'default'
-                buildlisty.append(buildlisty[-1])
-                histx = list()
-                histx.append(buildlistx[0])
-                histy = list()
-                histy.append(buildlisty[0])
-
-                i = 1
-                while i < len(buildlisty):
-                    # insert new x,y
-                    histx.append(buildlistx[i])
-                    histy.append(buildlisty[i-1])
-
-                    # insert old x,y
-                    histx.append(buildlistx[i])
-                    histy.append(buildlisty[i])
-
-                    i += 1
-
-                current.x = np.array(histx)
-                current.y = np.array(histy)
-            else:
-                current.x = np.array(buildlistx)
-                current.y = np.array(buildlisty)
+                    if matches and match_count >= matches:
+                        break
 
-            curvelist.append(current)
+        # Append the last curve that we built
+        if current:
+            curve_list.append(bundle_curve(current, build_list_x, build_list_y))
 
-        f.close()
     except IOError:
-        print('could not load file: ' + fname)
+        print('could not load file: {}'.format(file_name))
         if verbose:
             traceback.print_exc(file=sys.stdout)
     except ValueError:
-        print('invalid pydv file: ' + fname)
+        print('invalid pydv file: {}'.format(file_name))
         if verbose:
             traceback.print_exc(file=sys.stdout)
 
-    return curvelist
+    return curve_list
 
 
 def filtercurves(curvelist, pattern):

pydv/scripts/date.txt

@@ -1 +1 @@
-05.23.2022
+08.11.2022

pydv/scripts/version.txt

@@ -1 +1 @@
-3.1.1
+3.1.2

setup.py

@@ -13,7 +13,7 @@
 
 setup(
         name='PyDV',
-        version='3.1.1',
+        version='3.1.2',
         description='PyDV: Python Data Visualizer',
         long_description=long_description,
         author='Edward Rusu',