Add total summary at top of status web page across all map instances (#…

…2372) * Adds total summaries at top of status web page across all map instances * Avoid division by zero + consistent HTML placing.
RocketMap · Jan 26, 2018 · 0a583ed · 0a583ed
1 parent e866af0
commit 0a583ed
Show file tree

Hide file tree

Showing 4 changed files with 135 additions and 13 deletions.
diff --git a/pogom/models.py b/pogom/models.py
@@ -40,7 +40,7 @@
 flaskDb = FlaskDB()
 cache = TTLCache(maxsize=100, ttl=60 * 5)
 
-db_schema_version = 22
+db_schema_version = 23
 
 
 class MyRetryDB(RetryOperationalError, PooledMySQLDatabase):
@@ -1033,6 +1033,13 @@ class MainWorker(BaseModel):
     accounts_working = IntegerField()
     accounts_captcha = IntegerField()
     accounts_failed = IntegerField()
+    success = IntegerField(default=0)
+    fail = IntegerField(default=0)
+    empty = IntegerField(default=0)
+    skip = IntegerField(default=0)
+    captcha = IntegerField(default=0)
+    start = IntegerField(default=0)
+    elapsed = IntegerField(default=0)
 
     @staticmethod
     def get_account_stats():
@@ -3078,6 +3085,10 @@ def database_migrate(db, old_ver):
                        'ADD CONSTRAINT CONSTRAINT_4 CHECK ' +
                        '(`latest_seen` <= 3600);')
 
+    if old_ver < 23:
+        db.drop_tables([WorkerStatus])
+        db.drop_tables([MainWorker])
+
     # Always log that we're done.
     log.info('Schema upgrade complete.')
     return True
diff --git a/pogom/search.py b/pogom/search.py
@@ -313,7 +313,14 @@ def worker_status_db_thread(threads_status, name, db_updates_queue):
                     'last_modified': datetime.utcnow(),
                     'accounts_working': status['active_accounts'],
                     'accounts_captcha': status['accounts_captcha'],
-                    'accounts_failed': status['accounts_failed']
+                    'accounts_failed': status['accounts_failed'],
+                    'success': status['success_total'],
+                    'fail': status['fail_total'],
+                    'empty': status['empty_total'],
+                    'skip': status['skip_total'],
+                    'captcha': status['captcha_total'],
+                    'start': status['starttime'],
+                    'elapsed': status['elapsed']
                 }
             elif status['type'] == 'Worker':
                 workers[status['username']] = WorkerStatus.db_format(
@@ -380,6 +387,7 @@ def search_overseer_thread(args, new_location_queue, control_flags, heartb,
         'success_total': 0,
         'fail_total': 0,
         'empty_total': 0,
+        'elapsed': 0,
         'scheduler': args.scheduler,
         'scheduler_status': {'tth_found': 0}
     }
@@ -622,6 +630,7 @@ def get_stats_message(threadStatus, search_items_queue_array, db_updates_queue,
     if elapsed == 0:
         elapsed = 1
 
+    overseer['elapsed'] = elapsed
     sph = overseer['success_total'] * 3600.0 / elapsed
     fph = overseer['fail_total'] * 3600.0 / elapsed
     eph = overseer['empty_total'] * 3600.0 / elapsed
@@ -648,11 +657,12 @@ def get_stats_message(threadStatus, search_items_queue_array, db_updates_queue,
     message += (
         'Total active: {}  |  Success: {} ({:.1f}/hr) | ' +
         'Fails: {} ({:.1f}/hr) | Empties: {} ({:.1f}/hr) | ' +
-        'Skips {} ({:.1f}/hr) | Captchas: {} ({:.1f}/hr)|${:.5f}/hr|${:.3f}/mo'
+        'Skips {} ({:.1f}/hr) | Captchas: {} ({:.1f}/hr) (${:.1f}/hr, ' +
+        '${:.1f}/mo) | Elapsed: {:.1f}h'
     ).format(overseer['active_accounts'], overseer['success_total'], sph,
              overseer['fail_total'], fph, overseer['empty_total'], eph,
              overseer['skip_total'], skph, overseer['captcha_total'], cph,
-             ccost, cmonth)
+             ccost, cmonth, elapsed / 3600.0)
     return message
 
 

diff --git a/static/js/status.js b/static/js/status.js
@@ -5,6 +5,24 @@ var showHashTable = true
 var showInstances = true
 var showWorkers = true
 var hashkeys = {}
+var statshash = 'summarystats'  /* unique statistics worker name */
+var active
+var success
+var failed
+var empty
+var skipped
+var captcha
+var mainWorkers
+var elapsedTotal
+var elapsedSecs
+var elapsedHours
+var successPerHour
+var failsPerHour
+var emptyPerHour
+var skippedPerHour
+var captchasPerHour
+var captchasCost
+var captchasCostMonthly
 
 // Raw data updating
 var minUpdateDelay = 1000 // Minimum delay between updates (in ms).
@@ -90,7 +108,7 @@ function addHashtable(mainKeyHash, keyHash) {
 }
 
 function processWorker(i, worker) {
-    var hash = hashFnv32a(worker['username'], true)
+    const hash = hashFnv32a(worker['username'], true)
     var mainWorkerHash
     if (showWorkers && showInstances) {
         mainWorkerHash = hashFnv32a(worker['worker_name'], true)
@@ -108,7 +126,7 @@ function processWorker(i, worker) {
         addWorker(mainWorkerHash, hash)
     }
 
-    var lastModified = getFormattedDate(new Date(worker['last_modified']))
+    const lastModified = getFormattedDate(new Date(worker['last_modified']))
 
     $('#username_' + hash).html(worker['username'])
     $('#success_' + hash).html(worker['success'])
@@ -121,15 +139,15 @@ function processWorker(i, worker) {
 }
 
 function processHashKeys(i, hashkey) {
-    var key = hashkey['key']
-    var keyHash = hashFnv32a(key, true)
+    const key = hashkey['key']
+    const keyHash = hashFnv32a(key, true)
     if ($('#hashtable_global').length === 0) {
         createHashTable('global')
     }
 
     if ($('#hashrow_' + keyHash).length === 0) {
         addHashtable('global', keyHash)
-        var keyValues = {
+        const keyValues = {
             samples: [],
             nextSampleIndex: 0
         }
@@ -138,18 +156,18 @@ function processHashKeys(i, hashkey) {
     }
 
     // Calculate average value for Hash keys.
-    var writeIndex = hashkeys[key].nextSampleIndex % 60
+    const writeIndex = hashkeys[key].nextSampleIndex % 60
     hashkeys[key].nextSampleIndex += 1
     hashkeys[key].samples[writeIndex] = hashkey['maximum'] - hashkey['remaining']
-    var numSamples = hashkeys[key].samples.length
+    const numSamples = hashkeys[key].samples.length
     var sumSamples = 0
     for (var j = 0; j < numSamples; j++) {
         sumSamples += hashkeys[key].samples[j]
     }
 
-    var usage = sumSamples / Math.max(numSamples, 1) // Avoid division by zero.
+    const usage = sumSamples / Math.max(numSamples, 1) // Avoid division by zero.
 
-    var lastUpdated = getFormattedDate(new Date(hashkey['last_updated']))
+    const lastUpdated = getFormattedDate(new Date(hashkey['last_updated']))
     var expires = getFormattedDate(new Date(hashkey['expires']))
     if (!moment(expires).unix()) {
         expires = 'Unknown/Invalid'
@@ -167,6 +185,7 @@ function processHashKeys(i, hashkey) {
 }
 
 function parseResult(result) {
+    addTotalStats(result)
     if (showInstances) {
         $.each(result.main_workers, processMainWorker)
     }
@@ -322,6 +341,87 @@ function updateStatus() {
     })
 }
 
+/*
+ * Generate Statistics Across All Workers
+ */
+function addStatsWorker(hash) {
+    var worker = `
+    <div id="worker_${hash}" class="worker">
+    <span id="name_${hash}" class="name"></span>
+    <span id="method_${hash}" class="method"></span>
+    <span id="message_${hash}" class="message"></span>
+    </div>
+    `
+
+    $('#stats_worker').html(worker)
+}
+
+function getStats(i, worker) {
+    success += worker['success']
+    failed += worker['fail']
+    empty += worker['empty']
+    skipped += worker['skip']
+    captcha += worker['captcha']
+    mainWorkers += 1
+
+    elapsedTotal += worker['elapsed']
+    elapsedSecs = elapsedTotal / (i + 1)
+    elapsedHours = elapsedSecs / 3600
+}
+
+function addTotalStats(result) {
+    var statmsg, title
+
+    active = 0
+    success = 0
+    failed = 0
+    empty = 0
+    skipped = 0
+    captcha = 0
+    mainWorkers = 0
+    elapsedTotal = 0
+    elapsedSecs = 0
+    elapsedHours = 0
+    successPerHour = 0
+    failsPerHour = 0
+    emptyPerHour = 0
+    skippedPerHour = 0
+    captchasPerHour = 0
+    captchasCost = 0
+    captchasCostMonthly = 0
+
+    $.each(result.main_workers, getStats)
+
+    if ((mainWorkers > 1) || !(showWorkers && showInstances)) {
+        active += result.workers.length
+
+        // Avoid division by zero.
+        elapsedSecs = Math.max(elapsedSecs, 1)
+
+        successPerHour = (success * 3600 / elapsedSecs) || 0
+        failsPerHour = (failed * 3600 / elapsedSecs) || 0
+        emptyPerHour = (empty * 3600 / elapsedSecs) || 0
+        skippedPerHour = (skipped * 3600 / elapsedSecs) || 0
+        captchasPerHour = (captcha * 3600 / elapsedSecs) || 0
+        captchasCost = captchasPerHour * 0.00299
+        captchasCostMonthly = captchasCost * 730
+
+        if ($('#worker_' + statshash).length === 0) {
+            addStatsWorker(statshash)
+        }
+
+        statmsg = 'Total active: ' + active + ' | Success: ' + success.toFixed() + ' (' + successPerHour.toFixed(1) + '/hr) | Fails: ' + failed.toFixed() + ' (' + failsPerHour.toFixed(1) + '/hr) | Empties: ' + empty.toFixed() + ' (' + emptyPerHour.toFixed(1) + '/hr) | Skips: ' + skipped.toFixed() + ' (' + skippedPerHour.toFixed(1) + '/hr) | Captchas: ' + captcha.toFixed() + ' (' + captchasPerHour.toFixed(1) + '/hr) ($' + captchasCost.toFixed(1) + '/hr, $' + captchasCostMonthly.toFixed(1) + '/mo) | Elapsed:  ' + elapsedHours.toFixed(1) + 'h<hr />'
+        if (mainWorkers > 1) {
+            title = '(Total Statistics across ' + mainWorkers + ' instances)'
+        } else {
+            title = '(Total Statistics across ' + mainWorkers + ' instance)'
+        }
+        $('#name_' + statshash).html('All Instances')
+        $('#method_' + statshash).html(title)
+        $('#message_' + statshash).html(statmsg)
+    }
+}
+
 /**
  * Calculate a 32 bit FNV-1a hash
  * Found here: https://gist.github.com/vaiorabbit/5657561

diff --git a/templates/status.html b/templates/status.html
@@ -82,6 +82,7 @@ <h3>Hash Key Status</h3>
             <button>login</button>
         </form>
     </div>
+    <div id="stats_worker"></div>
 </div>
 <!-- Load JS libs before custom scripts. -->
 <script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.18.1/moment-with-locales.min.js"></script>