feat: add --exclude flag

deepsea_ai/__main__.py

@@ -96,14 +96,16 @@ def setup_command(config, mirror):
               help='Path to the folder with video files to upload. These can be either mp4 or mov files that '
                    'ffmpeg '
                    'understands.')
+@click.option('-e', '--exclude', type=str, multiple=True,
+              help='Exclude directory or file. Excludes any directory or file that contains the given string')
 @click.option('--cluster', type=str, required=True,
               help='Name of the cluster to use to batch process.  This must correspond to an available Elastic '
                    'Container Service cluster.')
 @click.option('--job', type=str, required=True,
               help='Name of the job, e.g. DiveV4361 benthic outline')
 @click.option('--conf-thres', type=click.FLOAT, default=.01, help='Confidence threshold for the model')
 @click.option('--iou-thres', type=click.FLOAT, default=.1, help='IOU threshold for the model')
-def batchprocess_command(config, check, upload, clean, cluster, job, input, conf_thres, iou_thres):
+def batchprocess_command(config, check, upload, clean, cluster, job, input, exclude, conf_thres, iou_thres):
     """
      (optional) upload, then batch process in an ECS cluster
     """
@@ -115,7 +117,7 @@ def batchprocess_command(config, check, upload, clean, cluster, job, input, conf
     input_path = Path(input)
     resources = custom_config.get_resources(cluster)
     user_name = custom_config.get_username()
-    videos = custom_config.check_videos(input_path)
+    videos = custom_config.check_videos(input_path, exclude)
     tags = custom_config.get_tags(f'Video uploaded from {input} by user {user_name} ')
 
     for v in videos:
@@ -147,6 +149,8 @@ def batchprocess_command(config, check, upload, clean, cluster, job, input, conf
 @click.option('-i', '--input', type=str, required=True,
               help='Path to the folder with video files to upload. These can be either mp4 or mov files that '
                    'ffmpeg understands.')
+@click.option('-e', '--exclude', type=str, multiple=True,
+              help='Exclude directory or file. Excludes any directory or file that contains the given string')
 @click.option('--input-s3', type=str, required=True,
               help=f'Path to the s3 bucket with video files. These can be either mp4 or '
                                                f'mov files that ffmpeg understands, e.g. s3://{example_input_process_s3}')
@@ -164,7 +168,7 @@ def batchprocess_command(config, check, upload, clean, cluster, job, input, conf
 @click.option('-s', '--save-vid', is_flag=True, default=False,
               help='Set option to output original video with detection boxes overlaid.')
 @click.option('--instance-type', type=str, default='ml.g4dn.xlarge', help='AWS instance type, e.g. ml.g4dn.xlarge, ml.c5.xlarge')
-def process_command(config, tracker, input, input_s3, output_s3, model_s3, config_s3, model_size, reid_model_url,
+def process_command(config, tracker, input, exclude, input_s3, output_s3, model_s3, config_s3, model_size, reid_model_url,
                     conf_thres, iou_thres, save_vid, job_description, instance_type):
     """
      upload video(s) then process with a model
@@ -184,7 +188,7 @@ def process_command(config, tracker, input, input_s3, output_s3, model_s3, confi
 
     if bucket.create(input_s3, tags) and bucket.create(output_s3, tags): 
 
-        videos = custom_config.check_videos(input_path)
+        videos = custom_config.check_videos(input_path, exclude)
         input_s3, size_gb = upload_tag.video_data(videos, input_s3, tags)
 
         # insert the datetime prefix to make a unique key for the output

deepsea_ai/config/config.py

@@ -169,15 +169,27 @@ def get_resources(self, stack_name: str) -> dict:
         return None
 
     staticmethod
-    def check_videos(self, input_path: Path) -> List[Path]:
+    def check_videos(self, input_path: Path, exclude: tuple) -> List[Path]:
         """
          Check for videos with acceptable suffixes and return the Paths to them
         :param input_path: input path to search (non-recursively)
+        :param exclude: directory or files to exclude from the list of videos to process
         :return:
         """
         vid_formats = ['.mov', '.avi', '.mp4', '.mpg', '.mpeg', '.m4v', '.wmv', '.mkv']  # acceptable video suffixes
-        files = sorted(input_path.glob('**/*'))
-        videos = [x for x in files if x.suffix.lower() in vid_formats and '._' not in x.name]
+
+        # convert exclude tuple to list
+        excludes = list(exclude)
+        print(f'Excluding any file or directory that contains {excludes}')
+
+        def search(x:Path):
+            if excludes:
+                found = [ x.name.__contains__(e) for e in excludes ]
+                return (True not in found and x.suffix.lower() in vid_formats and '._' not in x.name)
+            else:
+                return (x.suffix.lower() in vid_formats and '._' not in x.name)
+
+        videos = [x for x in input_path.glob("**/*") if search(x)]
         num_videos = len(videos)
         assert (num_videos > 0), "No videos to process"
         video_paths = [Path(x) for x in videos]

docs/commands/process.md

@@ -53,3 +53,16 @@ To process videos in a directory with the job name *"DocRickets dive 1423"* in y
 ```
 deepsea-ai ecsprocess -u -c benthic33k -j "DocRickets dive 1423" -i /Volumes/M3/mezzanine/DocRicketts/2022/02/1423/ 
 ```
+
+To process videos in a directory with the job name "DocRicketts 2021/08 with a cluster called mbari315k model", excluding any dives with the name D1371, D1374, or D1375
+
+```
+deepsea-ai ecsprocess -u \
+        --job "DocRicketts 2021/08 with mbari315k model" \
+        --cluster strongsort-yolov5-mbari315k  \
+        --config 902005prod.ini \
+        --conf-thres 0.2 \
+        --iou-thres 0.2 \
+        --input /Volumes/M3/mezzanine/DocRicketts/2021/08/ \
+        --exclude D1371 --exclude D1374 --exclude D1375
+```