Speed up resetting procedure (#276)

* Move and then delete

* move state folders into a subdirectory of state

so that it can be removed without getting resource busy error

Co-authored-by: ben foley <ben@cbmm.io>

docs/wiki/viewing-elpis-training-log-file.md

@@ -34,9 +34,9 @@ docker exec -it CONTAINER_ID bash
 5. Or use this one-liner: `docker exec -it $(docker ps -q) bash`
 
 
-6. Look in the `/state/models` directory. The hashes are directories of the models that have been made. Change into the current model dir. 
+6. Look in the `/state/of_origin/models` directory. The hashes are directories of the models that have been made. Change into the current model dir. 
 ```shell
-cd /state/models
+cd /state/of_origin/models
 ls
 cd XXXX
 ```

elpis/__init__.py

@@ -64,7 +64,7 @@ def create_app(test_config=None):
     # the app.config, however, this would need to change for multi-user.
     # Each user would require a unique Interface. One Interface
     # stores all the artifacts that user has generated.
-    interface_path = Path(os.path.join(elpis_path, '/state'))
+    interface_path = Path(os.path.join(elpis_path, '/state/of_origin'))
     if not interface_path.exists():
         app.config['INTERFACE'] = Interface(interface_path)
     else:
@@ -90,7 +90,7 @@ def create_app(test_config=None):
             print('Tensorboard is not running, start it')
             tensorboard = program.TensorBoard()
             tensorboard.configure(argv=['tensorboard',
-                                        '--logdir=/state/models',
+                                        '--logdir=/state/of_origin/models',
                                         '--port=6006',
                                         '--host=0.0.0.0'])
             url = tensorboard.launch()

elpis/engines/common/objects/interface.py

@@ -2,6 +2,7 @@
 import os
 from pathlib import Path
 import shutil
+import subprocess
 import re
 
 from appdirs import user_data_dir
@@ -52,7 +53,7 @@ class InvalidInterfaceError(Exception):
                 and path.is_dir()
                 and config_file_path.exists()
                 and config_file_path.is_file()):
-                # a valid interface exists. (this is a shallow check)
+                    # a valid interface exists. (this is a shallow check)
                     pass
             else:
                 raise InvalidInterfaceError
@@ -61,15 +62,10 @@ class InvalidInterfaceError(Exception):
         except InvalidInterfaceError:
             # Must wipe the interface and make a new one
             if path.exists():
-                # Tempted to use shutil.rmtree? It breaks if we have mounted /state from
-                # local filesystem into the docker container.
-                # Error is "Device or resource busy: '/state'"
-                # We need to keep the dir and delete the contents...
-                for root, subdirectories, files in os.walk(path):
-                    for file_ in files:
-                        os.unlink(os.path.join(root, file_))
-                    for directory in subdirectories:
-                        shutil.rmtree(os.path.join(root, directory))
+                deletion_path = Path('/state/tmp')
+                path.rename(deletion_path)
+                # Delete in the background
+                subprocess.Popen(["rm", "-r", deletion_path])
 
             super().__init__(
                 parent_path=path.parent,

elpis/examples/cli/hft/create_dataset.py

@@ -34,7 +34,7 @@ def main(dataset_name: str, reset: bool):
     # ======
     # Use or create the Elpis interface directory where all the associated files/objects are stored.
     print('Create interface')
-    elpis = Interface(path=Path('/state'), use_existing=reset)
+    elpis = Interface(path=Path('/state/of_origin'), use_existing=reset)
 
     # Step 1
     # ======

elpis/examples/cli/hft/train.py

@@ -30,7 +30,7 @@ def main(dataset_name: str, reset: bool):
     # ======
     # Use or create the Elpis interface directory where all the associated files/objects are stored.
     print('Create interface')
-    elpis = Interface(path=Path('/state'), use_existing=reset)
+    elpis = Interface(path=Path('/state/of_origin'), use_existing=reset)
 
     # Step 1
     # ======
@@ -72,9 +72,9 @@ def main(dataset_name: str, reset: bool):
     # TODO add model settings
     print('Linking dataset')
     model.link_dataset(dataset)
-    if Path('/state/models/latest').is_dir():
-        os.remove('/state/models/latest')
-    os.symlink(f'/state/models/{model.hash}', '/state/models/latest', target_is_directory=True)
+    if Path('/state/of_origin/models/latest').is_dir():
+        os.remove('/state/of_origin/models/latest')
+    os.symlink(f'/state/of_origin/models/{model.hash}', '/state/of_origin/models/latest', target_is_directory=True)
     print('Start training. This may take a while')
     model.train()
 

elpis/examples/cli/hft/transcribe.py

@@ -14,7 +14,7 @@ def main(model_name: str, infer_path: str):
     # ======
     # Use the Elpis interface directory where all the associated files/objects are stored.
     print('Create interface')
-    elpis = Interface(path=Path('/state'), use_existing=True)
+    elpis = Interface(path=Path('/state/of_origin'), use_existing=True)
 
     # Step 1
     # ======
@@ -44,10 +44,10 @@ def main(model_name: str, infer_path: str):
     print('Linking model')
     transcription.link(model)
 
-    if Path('/state/transcriptions/latest').is_dir():
-        os.remove('/state/transcriptions/latest')
-    os.symlink(f'/state/transcriptions/{transcription.hash}',
-               '/state/transcriptions/latest',
+    if Path('/state/of_origin/transcriptions/latest').is_dir():
+        os.remove('/state/of_origin/transcriptions/latest')
+    os.symlink(f'/state/of_origin/transcriptions/{transcription.hash}',
+               '/state/of_origin/transcriptions/latest',
                target_is_directory=True)
 
     print(f'Load audio from {infer_path}')

elpis/examples/cli/kaldi/train.py

@@ -35,7 +35,7 @@
 # ======
 # Create a Kaldi interface directory (where all the associated files/objects
 # will be stored).
-elpis = Interface(path=Path('/state'), use_existing=True)
+elpis = Interface(path=Path('/state/of_origin'), use_existing=True)
 
 
 # Step 1

elpis/examples/cli/kaldi/transcribe.py

@@ -8,7 +8,7 @@
 # ======
 # Create a Kaldi interface directory (where all the associated files/objects
 # will be stored).
-elpis = Interface(path='/state', use_existing=True)
+elpis = Interface(path='/state/of_origin', use_existing=True)
 
 # Step 1
 # ======