add printing logs to terminal as they are updated

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

README.md

@@ -284,8 +284,10 @@ $ tunel run-app waffles slurm/jupyter
 
 You'll see the above execute commands to interact with the launcher, and then go into an exponential backoff while
 waiting for a node. When finished, the above will launch a job with an interactive notebook and return the connection
-information when it's ready (not done yet). Generally this command will find the app.yaml under apps/slurm/jupyter in the default directory.
-Each app.yaml will define it's own launcher and other needs for running.
+information. Note that you should watch the error and output logs to determine when the application is ready to connect to. E.g.,
+a Singularity container will likely need to be pulled, and then converted to SIF, which unfortunately isn't quick. 
+When it's ready, try connecting. This command generally works by finding the app.yaml under apps/slurm/jupyter in the default directory,
+an each app.yaml will define it's own launcher and other needs for running.
 
 ## TODO
 

tunel/launcher/slurm.py

@@ -6,6 +6,7 @@
 from tunel.launcher.base import Launcher
 import tunel.utils as utils
 import tunel.ssh
+import threading
 import time
 import os
 
@@ -180,10 +181,14 @@ def show_logs_instruction(self, logs_prefix):
         logger.info("ssh %s cat %s.err" % (self.ssh.server, logs_prefix))
         print()
 
-    def show_logs(self, logs_prefix):
-        self.ssh.execute("ssh %s cat %s.out" % (self.ssh.server, logs_prefix))
-        self.ssh.execute("ssh %s cat %s.err" % (self.ssh.server, logs_prefix))
-        print()
+    def print_updated_logs(self, logs_prefix):
+        """
+        Start a separate thread that regularly checks and prints logs (when there is an updated line)
+        """
+        logs_thread = threading.Thread(
+            target=print_logs, name="Logger", args=[self.ssh, logs_prefix]
+        )
+        logs_thread.start()
 
     def run(self, cmd, job_name=None, logs_prefix=None):
         """
@@ -212,17 +217,16 @@ def run(self, cmd, job_name=None, logs_prefix=None):
                 # 1. Show the user how to quickly get logs (if logs_prefix provided)
                 if logs_prefix:
                     self.show_logs_instruction(logs_prefix)
+
                 machine = self.get_machine(job_name)
                 logger.info("%s is running on %s!" % (job_name, machine))
 
                 # Setup port forwarding
                 time.sleep(10)
                 print("== Connecting to %s ==" % job_name)
-                if logs_prefix:
-                    self.show_logs(logs_prefix)
                 print("== Instructions ==")
                 print(
-                    "1. Password, output, and error printed to this terminal? Look at logs (see instruction above)"
+                    "1. Password, output, and error will print to this - make sure application is ready before interaction."
                 )
                 print(
                     "2. Browser: http://%s:%s/ -> http://localhost:%s/..."
@@ -232,4 +236,34 @@ def run(self, cmd, job_name=None, logs_prefix=None):
                     "3. To end session: tunel stop-slurm %s %s"
                     % (self.ssh.server, job_name)
                 )
+                # Create another process to check logs?
+                if logs_prefix:
+                    self.print_updated_logs(logs_prefix)
                 self.ssh.tunnel(machine)
+
+
+def print_logs(ssh, logs_prefix):
+
+    # Output and error commands
+    output_command = "ssh %s tail -3 %s.out" % (ssh.server, logs_prefix)
+    error_command = "ssh %s tail -3 %s.err" % (ssh.server, logs_prefix)
+
+    last_out = None
+    last_err = None
+    while True:
+        time.sleep(5)
+        new_out = ssh.execute_or_fail(output_command, quiet=True)
+        new_err = ssh.execute_or_fail(error_command, quiet=True)
+        changed = False
+        if new_out != last_out:
+            logger.info("\n" + output_command)
+            print(new_out)
+            last_out = new_out
+            changed = True
+        if new_err != last_err:
+            logger.error("\n" + error_command)
+            print(new_err)
+            last_err = new_err
+            changed = True
+        if changed:
+            print()

tunel/ssh.py

@@ -60,14 +60,14 @@ def __repr__(self):
     def __str__(self):
         return "[tunel-ssh]"
 
-    def execute(self, cmd, login_shell=False):
+    def execute(self, cmd, login_shell=False, quiet=False):
         """
         Execute a command via ssh and a known, named connection.
         """
         if not isinstance(cmd, list):
             cmd = shlex.split(cmd)
         cmd = ["ssh", self.server] + cmd
-        return tunel.utils.run_command(cmd)
+        return tunel.utils.run_command(cmd, quiet=quiet)
 
     def scp_to(self, src, dest):
         """
@@ -93,11 +93,11 @@ def print_output(self, output, success_code=0):
         else:
             print(output["message"].strip())
 
-    def execute_or_fail(self, cmd, success_code=0):
+    def execute_or_fail(self, cmd, success_code=0, quiet=False):
         """
         Execute a command, show the command preview, only continue on success.
         """
-        output = self.execute(cmd)
+        output = self.execute(cmd, quiet=quiet)
         if output["return_code"] != success_code:
             logger.exit(output["message"])
         return output["message"].strip()

tunel/utils/terminal.py

@@ -50,7 +50,7 @@ def get_installdir():
     return os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 
 
-def run_command(cmd, stream=False):
+def run_command(cmd, stream=False, quiet=False):
     """run_command uses subprocess to send a command to the terminal.
 
     Parameters
@@ -60,7 +60,8 @@ def run_command(cmd, stream=False):
     if none specified, will alert that command failed.
 
     """
-    logger.info(" ".join(cmd))
+    if not quiet:
+        logger.info(" ".join(cmd))
     stdout = PIPE if not stream else None
     output = Popen(cmd, stderr=STDOUT, stdout=stdout)
     t = output.communicate()[0], output.returncode