testpod works!!!

Docker/Dockerfile

@@ -236,8 +236,14 @@ COPY ./abcd/keyringrc.cfg /.config/python_keyring/
 
 # Kubernetes jobs don't let you specify a user to run as, so we need
 # to create it beforehand
-RUN adduser --system --group --no-create-home nonroot
-USER nonroot
+ARG USERNAME=nonroot
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+
+RUN groupadd --gid $USER_GID $USERNAME \
+    && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME
+
+USER $USERNAME
 
 WORKDIR "/fastsurfer"
 # ENTRYPOINT ["/fastsurfer/Docker/entrypoint.sh","/fastsurfer/run_fastsurfer.sh"]

abcd/ABCDfastPipe_Docker.py

@@ -44,7 +44,7 @@ def make_parser() -> argparse.ArgumentParser:
     parser.add_argument(
         "-s", "--sbjid_time",
         dest="rawsubjses_str",
-        required=False,
+        required=True,
         help="""The underscore separated pairing of the subject ID and timepoint (e.g., NDARINV00XXXXY_baselineYear1Arm1) representing
             the subject-timepoint pair to be processed.""")
 
@@ -66,12 +66,14 @@ def make_parser() -> argparse.ArgumentParser:
         default="jrusse10",
         help="""NDA username (NOT miNDAR or Login.gov).""")
 
-    parser.add_argument(
-        "-p", "--packageID",
-        dest="miNDAR_packageID",
-        required=True,
-        help="""Package ID for the miNDAR containing the scans to be processed. \
-            This is equivalent to the last seven characters of the miNDAR username.""")
+    # For some reason I can't currently figure out, including this argument causes 
+    # miNDAR_packageID to be set to None when passed to the Downloader
+    # parser.add_argument(
+    #     "-p", "--packageID",
+    #     dest="miNDAR_packageID",
+    #     required=False,
+    #     help="""Package ID for the miNDAR containing the scans to be processed. \
+    #         This is equivalent to the last seven characters of the miNDAR username.""")
 
     # parser.add_argument(
     #     "-t", "--table",
@@ -109,13 +111,13 @@ def make_parser() -> argparse.ArgumentParser:
     parser.add_argument(
         "--aws_accesskey",
         dest="aws_access_key_id",
-        required=True,
+        required=False,
         help="""AWS access key for the S3 bucket.""")
 
     parser.add_argument(
         "--aws_secretkey",
         dest="aws_secret_access_key",
-        required=True,
+        required=False,
         help="""AWS secret key for the S3 bucket.""")
 
     return (parser)
@@ -154,7 +156,9 @@ def download(self):
         # Downloading files specific to this subject_ses from NDA
         logTitle(f'Downloading and Unpacking Scan Files for {self.rawsubjses_str}')
         try:
-            Downloader(self.rawsubjses_str, self.input_dir, self.NDA_username, self.miNDAR_packageID, self.nThreads)
+            Downloader(rawsubjses_str = self.rawsubjses_str, input_dir = self.input_dir, 
+                       NDA_username = self.NDA_username, miNDAR_packageID = self.miNDAR_packageID, 
+                       nThreads = self.nThreads)
         except RuntimeError as e:
             print(e)
         return self.rawsubjses_str
@@ -191,34 +195,6 @@ def upload(self):
                  secret_access_key=self.aws_secret_access_key, s3bucket=self.aws_s3_bucket)
 
 
-def msgPull():
-    #!/usr/bin/env python
-    import pika
-    import time
-
-    connection = pika.BlockingConnection(
-        pika.ConnectionParameters(host='10.97.22.234'))
-    channel = connection.channel()
-
-    channel.queue_declare(queue='fastqueue', durable=True)
-    print(' [*] Waiting for messages. To exit press CTRL+C')
-
-    def callback(ch, method, properties, body):
-        print(f" [x] Received {body.decode()}")
-        time.sleep(body.count(b'.'))
-        getmsg(body)
-        print(" [x] Done")
-        ch.basic_ack(delivery_tag=method.delivery_tag)
-
-    def getmsg(body):
-        rawsubjses_str = body
-        return (rawsubjses_str)
-
-    channel.basic_qos(prefetch_count=1)
-    channel.basic_consume(queue='task_queue', on_message_callback=callback)
-    channel.start_consuming()
-
-
 def main(
         rawsubjses_str: str = None,
         input_dir: str = '/input',
@@ -227,18 +203,20 @@ def main(
         seg_only: bool = False,
         run_subseg: bool = False,
         NDA_username: str = "jrusse10",
-        miNDAR_packageID: str = 1226193,
-        aws_access_key_id: str = None,
-        aws_secret_access_key: str = None,
+        miNDAR_packageID: str = "1226193",
+        aws_access_key_id: str = 'AKIAQMJTZW6JRMGTOUCW',
+        aws_secret_access_key: str = 'Qv3kvwius5Z/QnyQH7K2CIkkHEhaO0IpjIiqAomP',
         aws_s3bucket: str = "brc.abcd"):
 
+
     # Need to replace with args
     keyring.set_password('nda-tools', 'jrusse10', '19Ireland61')
 
     try:
         FastPipe(rawsubjses_str, input_dir, log_dir, nThreads, seg_only, run_subseg, NDA_username,
                  miNDAR_packageID, aws_access_key_id, aws_secret_access_key, aws_s3bucket)
     except RuntimeError as e:
+        print(e)
         return e.args[0]
 
     return 0
@@ -247,6 +225,4 @@ def main(
 if __name__ == '__main__':
 
     arguments = make_parser().parse_args()
-    arguments.rawsubjses_str = sys.stdin.readlines()[0]
-
     sys.exit(main(**vars(arguments)))

abcd/fastSurferShell.py

@@ -20,9 +20,9 @@
 @dataclass
 class FastSurfer:
     rawsubjses_str: str
-    inputdir: str = '/input'
-    nThreads: str = 1
-    seg_only: bool = False
+    inputdir: str
+    nThreads: str
+    seg_only: bool
 
     def __post_init__(self):
         self.logger = logging.getLogger(__name__)

abcd/jobs/job_NDARINV005V6D2C_baselineYear1Arm1.yaml

@@ -6,17 +6,14 @@ metadata:
     jobgroup: fastjobs
 spec:
   template:
-    securityContext:
-      runAsUser: 1494709274
-      runAsGroup: 390800513
     metadata:
       name: fastjob
       labels:
         jobgroup: fastjob
     spec:
       containers:
       - name: abcdfast
-        image: jrussell9000/abcdfast:dev
+        image: docker.io/jrussell9000/abcdfast:032524
         args: ["-s", "NDARINV005V6D2C_baselineYear1Arm1"]
       imagePullSecrets:
       - name: regcred

abcd/kjob_tmpl.yaml

@@ -6,17 +6,14 @@ metadata:
     jobgroup: fastjobs
 spec:
   template:
-    securityContext:
-      runAsUser: 1494709274
-      runAsGroup: 390800513
     metadata:
       name: fastjob
       labels:
         jobgroup: fastjob
     spec:
       containers:
       - name: abcdfast
-        image: jrussell9000/abcdfast:dev
+        image: docker.io/jrussell9000/abcdfast:032524
         args: ["-s", "$ITEM"]
       imagePullSecrets:
       - name: regcred

abcd/makeJobs.py

@@ -14,7 +14,8 @@
 # Connection information for S3
 aws_service_name = 's3'
 aws_region_name = 'us-east-2'
-
+aws_access_key_id = 'AKIAQMJTZW6JRMGTOUCW'
+aws_secret_access_key = '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'
 aws_s3bucket = "brc.abcd"
 
 
@@ -33,6 +34,7 @@ def getSubjSesArray():
 df: pd.DataFrame = getSubjSesArray()
 
 for rawsubjses_str in df[1:2].values:
+    print(rawsubjses_str)
     search_text = "$ITEM"
 
     with open('kjob_tmpl.yaml', 'r') as file: 

abcd/ndaDownload.py

@@ -7,11 +7,11 @@
 from pathlib import Path
 from utils import suppress_stdout, setup_logger, run_command
 
-# Silence obnoxious messages from nda-tools
-with suppress_stdout():
-    from NDATools import prerun_checks_and_setup
-    from NDATools.Download import Download
-    from NDATools.Configuration import ClientConfiguration
+# # Silence obnoxious messages from nda-tools
+# with suppress_stdout():
+from NDATools import prerun_checks_and_setup
+from NDATools.Download import Download
+from NDATools.Configuration import ClientConfiguration
 
 '''
 This module provides programmatic access to the 'download' methods
@@ -51,11 +51,11 @@
 
 @dataclass
 class Downloader:
-    rawsubjses_str: str = None
-    input_dir: str = '/input'
-    NDA_username: str = None
-    miNDAR_packageID: str = None
-    nThreads: str = "1"
+    rawsubjses_str: str
+    input_dir: str
+    NDA_username: str
+    miNDAR_packageID: str
+    nThreads: str
 
     def __post_init__(self):
         self.main()
@@ -93,6 +93,8 @@ def configCheck(self):
 
         return (config)
 
+
+
     def download(self, config, args):
         # Initiate the download for this subject_ses
         rawsubj_str = self.rawsubjses_str.split("_")[0]
@@ -105,6 +107,7 @@ def download(self, config, args):
             s3Download.start()
 
         except Exception as exc:
+            print(exc)
             self.logger.exception(exc, exc_info=True)
 
     def unpack(self):

abcd/private-pod-example.yaml

@@ -5,8 +5,10 @@ metadata:
 spec:
   containers:
   - name: private-reg-container
-    image: jrussell9000/abcdfast:dev
+    image: docker.io/jrussell9000/abcdfast:032524
+    imagePullPolicy: Always
   imagePullSecrets:
   - name: regcred
 
 
+

abcd/testpod.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: testpod
+spec:
+  containers:
+  - name: test-pod-cont
+    image: docker.io/jrussell9000/abcdfast:032524
+    args: ["-s", "NDARINV003RTV85_baselineYear1Arm1"]
+    imagePullPolicy: Always
+    lifecycle: 
+      preStop:
+          exec:
+            command: ["/bin/sh","-c","cat /log/*"]
+