Skip to content

Commit

Permalink
Remove NoQueueMinCycleTime nonsense from sync worker
Browse files Browse the repository at this point in the history
  • Loading branch information
cpfair committed Oct 25, 2014
1 parent 2764fc0 commit bce9101
Showing 1 changed file with 2 additions and 9 deletions.
11 changes: 2 additions & 9 deletions sync_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@

Run = True
RecycleInterval = 2 # Time spent rebooting workers < time spent wrangling Python memory management.
NoQueueMinCycleTime = timedelta(seconds=30) # No need to hammer the database given the number of sync workers I have

oldCwd = os.getcwd()
WorkerVersion = subprocess.Popen(["git", "rev-parse", "HEAD"], stdout=subprocess.PIPE, cwd=os.path.dirname(__file__)).communicate()[0].strip()
Expand Down Expand Up @@ -47,18 +46,12 @@ def sync_heartbeat(state, user=None):

Sync.InitializeWorkerBindings()

sync_heartbeat("ready")

while Run:
cycleStart = datetime.utcnow() # Avoid having synchronization fall down during DST setback
processed_user_count = Sync.PerformGlobalSync(heartbeat_callback=sync_heartbeat, version=WorkerVersion)
RecycleInterval -= processed_user_count
# When there's no queue, all the workers sit sending 1000s of the queries to the database server
if processed_user_count == 0:
# Put this before the recycle shutdown, otherwise it'll quit and get rebooted ASAP
remaining_cycle_time = NoQueueMinCycleTime - (datetime.utcnow() - cycleStart)
if remaining_cycle_time > timedelta(0):
print("Pausing for %ss" % remaining_cycle_time.total_seconds())
sync_heartbeat("idle-spin")
time.sleep(remaining_cycle_time.total_seconds())
if RecycleInterval <= 0:
break
sync_heartbeat("idle")
Expand Down

0 comments on commit bce9101

Please sign in to comment.