moved template lib directories to "lib"

sim_data abstractions seem to be complete

.gitignore

@@ -27,7 +27,6 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
 lib64/
 node_modules/
 parts/

sirepo/importer.py

@@ -91,14 +91,14 @@ def read_zip(stream, sim_type=None):
                 zipped[b].write(c, 'wb')
         assert data, \
             'missing {} in archive'.format(simulation_db.SIMULATION_DATA_FILE)
-        needed = set
+        needed = set()
         s = sirepo.sim_data.get_class(data.simulationType)
         for n in s.lib_file_basenames(data):
 #TODO(robnagler) this does not allow overwrites of lib files,
-# but need to be modularlized
+# but it needs to be modularized
             if s.lib_file_exists(n):
                 continue
-#TODO(robnagler) raise useralert
+#TODO(robnagler) raise useralert instead of an assert
             assert n in zipped, \
                 'auxiliary file={} missing in archive'.format(n)
             needed.add(n)

sirepo/pkcli/srw.py

@@ -57,7 +57,7 @@ def f(file_type):
         return sim_data.srw_files_for_type(
             file_type,
             lambda f: PKDict(fileName=f.basename),
-            dir_path=sim_data.resource_dir(),
+            want_user_lib_dir=False,
         )
 
     p = sim_data.resource_path(srw_common.PREDEFINED_JSON)

sirepo/server.py

@@ -556,23 +556,25 @@ def api_updateFolder():
 
 @api_perm.require_user
 def api_uploadFile(simulation_type, simulation_id, file_type):
+    f = flask.request.files['file']
     sim = http_request.parse_params(
         file_type=file_type,
+        filename=f.filename,
         id=simulation_id,
         template=1,
         type=simulation_type,
     )
-    f = flask.request.files['file']
-    sim.filename = werkzeug.secure_filename(f.filename)
     e = None
     in_use = None
-    p = _lib_file_write_path(sim)
     with simulation_db.tmp_dir() as d:
         t = d.join(sim.filename)
         f.save(str(t))
         if hasattr(sim.template, 'validate_file'):
             e = sim.template.validate_file(sim.file_type, t)
-        if not e and p.check() and not flask.request.form.get('confirm'):
+        if (
+            not e and sim.sim_data.lib_file_exists(sim.filename)
+            and not flask.request.form.get('confirm')
+        ):
             in_use = _simulations_using_file(sim, ignore_sim_id=sim.id)
             if in_use:
                 e = 'File is in use in other simulations. Please confirm you would like to replace the file for all simulations.'
@@ -584,8 +586,7 @@ def api_uploadFile(simulation_type, simulation_id, file_type):
                 'fileType': sim.file_type,
                 'simulationId': sim.id,
             })
-        pkio.mkdir_parent_only(p)
-        t.rename(p)
+        t.rename(_lib_file_write_path(sim))
     return http_reply.gen_json({
         'filename': sim.filename,
         'fileType': sim.file_type,
@@ -680,7 +681,7 @@ def _simulation_data(res, path, data):
 def _simulations_using_file(sim, ignore_sim_id=None):
     res = []
     for r in simulation_db.iterate_simulation_datafiles(sim.type, _simulation_data):
-        if not sim.sim_data.is_file_used(r, _lib_file_write_path(sim).basename):
+        if not sim.sim_data.lib_file_in_use(r, sim.filename):
             continue
         s = r.models.simulation
         if s.simulationId == ignore_sim_id:

sirepo/sim_data/__init__.py

@@ -20,10 +20,6 @@
 import sirepo.util
 import sirepo.template
 
-
-#: root directory for template resources
-RESOURCE_DIR = pkio.py_path(pkresource.filename('template'))
-
 #: default compute_model
 _ANIMATION_NAME = 'animation'
 
@@ -69,6 +65,11 @@ def get_class(type_or_data):
     ).SimData
 
 
+def resource_dir():
+    """root directory for template resources"""
+    return pkio.py_path(pkresource.filename('template'))
+
+
 def template_globals(sim_type=None):
     """Initializer for templates
 
@@ -130,6 +131,7 @@ def compute_job_hash(cls, data):
         Returns:
             bytes: hash value
         """
+        cls._assert_server_side()
         m = data['models']
         res = hashlib.md5()
         for f in sorted(
@@ -154,8 +156,13 @@ def compute_job_hash(cls, data):
                     allow_nan=False,
                 )
             )
-        # lib_files already returns sorted list
-        res.update(''.join(str(f.mtime()) for f in cls.lib_files(data)).encode())
+        res.update(
+            ''.join(
+                (str(cls.lib_file_abspath(b).mtime()) for b in sorted(
+                    cls.lib_file_basenames(data))
+                ),
+            ).encode())
+
         return res.hexdigest()
 
     @classmethod
@@ -215,18 +222,6 @@ def frame_id(cls, data, response, model, index):
             ] + [str(m.get(k)) for k in cls._frame_id_fields(frame_args)],
         )
 
-    @classmethod
-    def is_file_used(cls, data, filename):
-        """Check if file in use by simulation
-
-        Args:
-            data (dict): simulation
-            filename (str): to check
-        Returns:
-            bool: True if `filename` in use by `data`
-        """
-        return any(f for f in cls.lib_file_basenames(data) if f == filename)
-
     @classmethod
     def is_parallel(cls, data):
         """Is this report a parallel (long) simulation?
@@ -253,16 +248,9 @@ def lib_file_abspath(cls, basename):
         Returns:
             object: py.path.local to files (duplicates removed) OR py.path.local
         """
-        from sirepo import simulation_db
-
-        res = []
-        for d in (
-            simulation_db.simulation_lib_dir(cls.sim_type()),
-            cls.resource_dir(),
-        ):
-            p = d.join(f)
-            if p.check(file=True):
-                return p
+        p = cls._lib_file_abspath(basename)
+        if p:
+            return p
         raise sirepo.util.UserAlert(
             'Simulation library file "{}" does not exist'.format(basename),
             'basename={} not in lib or resource directories',
@@ -282,33 +270,50 @@ def lib_file_basenames(cls, data):
 
     @classmethod
     def lib_file_exists(cls, basename):
-        return lib_file_abspath(basename).exists()
+        return bool(cls._lib_file_abspath(basename))
+
+    @classmethod
+    def lib_file_in_use(cls, data, basename):
+        """Check if file in use by simulation
+
+        Args:
+            data (dict): simulation
+            basename (str): to check
+        Returns:
+            bool: True if `basename` in use by `data`
+        """
+        return any(f for f in cls.lib_file_basenames(data) if f == basename)
 
     @classmethod
     def lib_file_name_with_model_field(cls, model_name, field, filename):
-        return '{}-{}.{}'.format(model_name, field, value)
+        return '{}-{}.{}'.format(model_name, field, filename)
 
     @classmethod
     def lib_file_name_with_type(cls, filename, file_type):
         return '{}.{}'.format(filename, filetype)
 
     @classmethod
-    def lib_file_name_without_type(cls, basename, file_type):
+    def lib_file_name_without_type(cls, basename):
         """Strip the file type prefix
 
         See `lib_file_name` which prefixes with ``model-field.``
 
         Args:
-            basename: path to file
-            file_type (str): type of file being searched for
+            basename: lib file name with type
         Returns:
             str: basename without type prefix
         """
-        return basename[len(file_type) + 1:]
+        return re.sub(r'^.*?-.*?\.', '', basename)
+
+    @classmethod
+    def lib_file_resource_dir(cls):
+        return cls._memoize(cls.resource_dir().join('lib'))
 
     @classmethod
     def lib_file_write_path(cls, basename):
         cls._assert_server_side()
+        from sirepo import simulation_db
+
         return simulation_db.simulation_lib_dir(cls.sim_type()).join(basename)
 
     @classmethod
@@ -322,12 +327,7 @@ def lib_files_for_extension(cls, ext):
         Returns:
             list: sorted list of absolute paths to lib files
         """
-        cls._assert_server_side()
-        from sirepo import simulation_db
-
-        return pkio.sorted_glob(
-            simulation_db.simulation_lib_dir(cls.sim_type()).join('*.{}'.format(ext)),
-        )
+        return cls._lib_file_list('*.{}'.format(ext))
 
     @classmethod
     def lib_files_for_type(cls, file_type):
@@ -338,15 +338,10 @@ def lib_files_for_type(cls, file_type):
         Returns:
             list: sorted list of files stripped of file_type
         """
-        cls._assert_server_side()
-        from sirepo import simulation_db
-
-        res = []
-        d = simulation_db.simulation_lib_dir(cls.sim_type())
-        for f in pkio.sorted_glob(d.join('{}.*'.format(file_type))):
-            if f.check(file=1):
-                res.append(cls._lib_file_name_without_type(f.basename, file_type))
-        return sorted(res)
+        return [
+            cls.lib_file_name_without_type(f) for f
+            in cls._lib_file_list('{}.*'.format(file_type))
+        ]
 
     @classmethod
     def lib_files_for_export(cls, data):
@@ -386,7 +381,7 @@ def lib_files_to_run_dir(cls, data, run_dir):
         from sirepo import simulation_db
 
         for b in cls.lib_file_basenames(data):
-            target_dir.join(b).mksymlinkto(
+            run_dir.join(b).mksymlinkto(
                 cls.lib_file_abspath(b),
                 absolute=False,
             )
@@ -475,25 +470,7 @@ def poll_seconds(cls, data):
 
     @classmethod
     def resource_dir(cls):
-        return cls._memoize(RESOURCE_DIR.join(cls.sim_type()))
-
-    @classmethod
-    def resource_files(cls):
-        """Files to copy for a new user
-
-        Returns:
-            list: path of resource files
-        """
-        return []
-
-    @classmethod
-    def resource_glob(cls, pattern):
-        """Match `pattern` in `resource_dir`
-
-        Returns:
-            patter: absolute path to folder
-        """
-        return pkio.sorted_glob(cls.resource_path(pattern))
+        return cls._memoize(resource_dir().join(cls.sim_type()))
 
     @classmethod
     def resource_path(cls, filename):
@@ -590,6 +567,34 @@ def _init_models(cls, models, names=None, dynamic=None):
                 dynamic=dynamic,
             )
 
+    @classmethod
+    def _lib_file_abspath(cls, basename):
+        from sirepo import simulation_db
+
+        for d in (
+            simulation_db.simulation_lib_dir(cls.sim_type()),
+            cls.lib_file_resource_dir(),
+        ):
+            p = d.join(basename)
+            if p.check(file=True):
+                return p
+        return None
+
+    @classmethod
+    def _lib_file_list(cls, pat, want_user_lib_dir=False):
+        cls._assert_server_side()
+        from sirepo import simulation_db
+
+        res = set()
+        x = [cls.lib_file_resource_dir()]
+        if want_user_lib_dir:
+            x.insert(0, simulation_db.simulation_lib_dir(cls.sim_type()))
+        for d in x:
+            res = res.union(
+                (f.basename for f in pkio.sorted_glob(d.join(pat))),
+            )
+        return sorted(res)
+
     @classmethod
     def _memoize(cls, value):
         """Cache class method (no args)
@@ -652,3 +657,5 @@ def _init():
     cfg = pkconfig.init(
         lib_file_uri=(None, str, 'where to get files from'),
     )
+
+_init()

sirepo/sim_data/elegant.py

@@ -66,10 +66,6 @@ def elegant_max_id(cls, data):
                     max_id = i
         return max_id
 
-    @classmethod
-    def resource_files(cls):
-        return cls.resource_glob('*.sdds')
-
     @classmethod
     def _compute_job_fields(cls, data, r, compute_model):
         res = []

sirepo/sim_data/jspec.py

@@ -71,10 +71,6 @@ def jspec_elegant_twiss_path(cls):
     def jspec_elegant_dir(cls):
         return simulation_db.simulation_dir('elegant')
 
-    @classmethod
-    def resource_files(cls):
-        return cls.resource_glob('*.tfs')
-
     @classmethod
     def _compute_job_fields(cls, data, r, compute_model):
         if r == 'rateCalculationReport':

sirepo/sim_data/rs4pi.py

@@ -53,10 +53,6 @@ def fixup_old_data(cls, data):
         dm.dicomAnimation4.setdefault('doseTransparency', 56)
 
 
-    @classmethod
-    def resource_files(cls):
-        return cls.resource_glob('beamlist*.txt')
-
     @classmethod
     def _compute_job_fields(cls, data, r, compute_model):
         if r == 'doseCalculation':

sirepo/sim_data/shadow.py

@@ -30,10 +30,6 @@ def fixup_old_data(cls, data):
                 cls.update_model_defaults(dm[m], 'watchpointReport')
         cls._organize_example(data)
 
-    @classmethod
-    def resource_files(cls):
-        return cls.resource_glob('*.txt')
-
     @classmethod
     def shadow_simulation_files(cls, data):
         m = data.models