Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

file 121 lines (94 sloc) 3.962 kb
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121
def main():
    """This is a simple tool to restart a process when it dies.

It's designed to restart aspen in development when it dies because files
have changed and you set changes_reload to 'yes'.

http://aspen.io/thrash/

"""
    try: # capture KeyboardInterrupt

        import os
        import signal
        import subprocess
        import sys
        import time

        # set unbuffered - http://stackoverflow.com/a/181654/253309
        sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

        if len(sys.argv) < 2:
            print ("usage: %s <child> [child opts and args]"
                   % os.path.basename(sys.argv[0]))
            sys.exit(1)

        BACKOFF_MIN = 0.10 # Start looping here.
        BACKOFF_MAX = 3.20 # Throttle back to here.
        PROMPT_AFTER = 60 # Give the user this much time to fix the error.
        INITIAL_WAIT = 15 # Give the user this much time to read the error.

        n = 0
        backoff = BACKOFF_MIN
        cumulative_time = 0

        while 1:

            # The child process exited.
            # =========================
            # Log restart attempts after the initial launch.

            n += 1
            backoff = min(backoff * 2, BACKOFF_MAX)
            if n > 1:
                m = "---- Restart #%s " % n
                print
                print m + ('-' * (79-len(m)))


            # Execute the child process.
            # ==========================
            # Then wait for it to return, dealing with INT.

            proc = subprocess.Popen( sys.argv[1:]
                                   , stdout=sys.stdout
                                   , stderr=sys.stderr
                                    )

            try:
                status = proc.wait()
                if status == 75: # signals INT in child
                    print "Received INT in child, apparently. Restarting."
                    continue
            except KeyboardInterrupt:
                try:
                    print ("Received INT in thrash while blocking on child, "
                           "sending HUP to child.")
                    time.sleep(0.2) # allow for second INT to reach thrash
                except KeyboardInterrupt:
                    print "Received second INT in thrash, exiting."
                    raise SystemExit

                proc.send_signal(signal.SIGHUP)
                
                # reset
                n = 0
                backoff = BACKOFF_MIN
                cumulative_time = 0

                continue


            # Decide how to proceed.
            # ======================

            if n == 1:
                # This is the first time we've thrashed. Give the user time to
                # parse the (presumed) traceback.
                cumulative_time += INITIAL_WAIT
                try:
                    time.sleep(INITIAL_WAIT)
                except KeyboardInterrupt:
                    # Allow user to fast-track this step.
                    
                    # reset
                    n = 0
                    backoff = BACKOFF_MIN
                    cumulative_time = 0

            elif cumulative_time < PROMPT_AFTER:
                # We've given the user time to parse the traceback. Now thrash
                # for a while.
                cumulative_time += backoff
                time.sleep(backoff)
                
            else:
                # We've been thrashing for a while. Pause.
                print
                try:
                    raw_input("Press any key to start thrashing again. ")
                except KeyboardInterrupt:
                    print

                # reset
                n = 0
                backoff = BACKOFF_MIN
                cumulative_time = 0


    except KeyboardInterrupt:
        time.sleep(0.1) # give child stdio time to flush
        print "Received INT in thrash, exiting."
Something went wrong with that request. Please try again.