pushing changes

app.py

@@ -53,6 +53,7 @@ def make_info():
         'count': p[2] if p[2] else 0, 
         'fill_color': unlocked_color.format(0.9) if p[3] == 0 else blocked_color.format(0.9)}
         for p in select_ip(orgs, ts_low, ts_high, only_locked=only_locked)]
+
     stats = [{'name': kind, 'color': color, 'pointStart': start, 'pointInterval': 24 * 3600 * 1000, 'data': stat} 
         for kind, color, start, stat in select_stats(orgs, ts_low, ts_high, only_locked=only_locked)]
 

ip_selector.py

@@ -68,7 +68,6 @@ def filter_ip(ip_dict, subnet_dict):
 
 def select_ip(orgs=[org for org in Org], ts_low=min_date, ts_high=max_date, use_cache=True, only_locked = False):
 	# sorry about that..
-	print(orgs)
 	if use_cache and len(orgs) == len(Org) and ts_low == min_date and ts_high == max_date and not only_locked and os.path.isfile(full_geo_cache):
 		with open(full_geo_cache, 'rb') as cache:
 			try:
@@ -95,7 +94,6 @@ def select_ip(orgs=[org for org in Org], ts_low=min_date, ts_high=max_date, use_
 		query += ' join block_geo on (block_id = blocked_ip.id) and (ip_subnet is null)'
 		query += where_clause(orgs, ts_low, ts_high, 'exclude_time')
 	query += ' order by time, type desc'
-	print(query)
 	data = engine.execute(query).fetchall()
 	return data
 
@@ -106,11 +104,34 @@ def select_stats(orgs=[org for org in Org], ts_low=min_date, ts_high=max_date, o
     query += ' order by date'
     data = engine.execute(query).fetchall()
 
-    start_ts = datetime.strptime(data[0]['date'], '%Y-%m-%d').timestamp() * 1000
-    stats = [('Заблокировано', '#FF0000', start_ts, [item['blocked'] if item['blocked'] else 0 for item in data])]
-    if not only_locked:
-    	stats.append(('Разблокировано', '#00FF00', start_ts, [item['unlocked'] if item['unlocked'] else 0 for item in data]))
-    return stats
+    def make_date(ts):
+        return datetime.fromtimestamp(ts // 1000).strftime('%Y-%m-%d')
+
+    def make_ts(formatted_date):
+        return datetime.strptime(formatted_date, '%Y-%m-%d').timestamp() * 1000
+
+    # fill empty days with zeros
+    def extend_stats(stats):
+        extended = []
+        for stat in stats:
+            if extended:
+                ts_start = make_ts(extended[-1]['date']) + 24 * 60 * 60 * 1000
+                ts_end = make_ts(stat['date'])
+                while ts_start < ts_end:
+                    extended.append({'date': make_date(ts_start), 'blocked': 0, 'unlocked': 0})
+                    ts_start += 24 * 60 * 60 * 1000
+            extended.append(stat)
+        return extended
+
+    if data:
+        #data = extend_stats(data)
+        start_ts = make_ts(data[0]['date'])
+        stats = [('Заблокировано', '#FF0000', start_ts, [item['blocked'] for item in data])]
+        if not only_locked:
+    	    stats.append(('Разблокировано', '#00FF00', start_ts, [item['unlocked'] for item in data]))
+        return stats
+    else:
+        return []
 
 
 def smart_print(orgs):

templates/index.html

@@ -71,12 +71,10 @@ <h5 align="center" style="padding-left:10px;color:#000">Воспользуйте
                     <div class="custom-control custom-checkbox"><label><input type="checkbox" class="custom-control-input" id="only_locked" name="only_locked">Только заблокированные</label></div>
                     <br>
                     <input type="submit" class="btn btn-info" id="submit_orgs" value="Фильтровать">
+                    <a href="https://yasobe.ru/na/na_server_blocked_here" role="button" class="btn btn-lg btn-success" style="margin-top: 40px">Поддержать авторов материально?</a>
                   </form>
                 </div>
               </div>
-               <div class="container">
-                 <a href="https://yasobe.ru/na/na_server_blocked_here" role="button" class="btn btn-lg btn-success" style="margin-top: 40px">Поддержать авторов материально?</a>
-               </div>
                </div>
             </div>
           </div>

update_from_repo.py

@@ -51,6 +51,10 @@ class DayStats:
     def __init__(self):
         self.blocked = defaultdict(int)
         self.unlocked = defaultdict(int)
+        self.no_org = 0
+
+    def __str__(self):
+        return '<DayStats> blocked: {}, unlocked: {}, not counted: {}'.format(sum(self.blocked.values()), sum(self.unlocked.values()), self.no_org)
 
 
 def get_changes(repo_path, squash=False):
@@ -128,7 +132,7 @@ def gen_clean_ips(repo):
         removed_ip_clean = removed_ip - added_ip
         logger_info.info('{} {} {} {} {} {}'.format(commit, date, len(added_ip), len(removed_ip), len(added_ip_clean), len(removed_ip_clean)))
         assert(len(added_ip) - len(added_ip_clean) == len(removed_ip) - len(removed_ip_clean))
-        yield date, commit, map(dict, added_ip_clean), map(dict, removed_ip_clean) 
+        yield date, commit, list(map(dict, added_ip_clean)), list(map(dict, removed_ip_clean)) 
 
 
 def update_geodata(session, added_ips, removed_ips, date, commit):
@@ -179,14 +183,20 @@ def update_geodata(session, added_ips, removed_ips, date, commit):
 
 def update_stats(session, added_ips, removed_ips, date, commit):
     stats = DayStats()
+    logger_info.info('added_ips: {}, removed_ips: {}, date: {}, commit: {}'.format(len(added_ips), len(removed_ips), date, commit))
+
     for added in added_ips:
         try:
             if added['ip']:
                 if added['org']:
                     stats.blocked[added['org']] += 1
+                else:
+                    stats.no_org += 1
             elif added['ip_subnet']:
                 if added['org']:
                     stats.blocked[added['org']] += count_network_ips(added['ip_subnet'])
+                else:
+                    stats.no_org += count_network_ips(added['ip_subnet'])
             else:
                 raise Exception("Bad ip data: " + str(added))
         except Exception as e:
@@ -197,14 +207,19 @@ def update_stats(session, added_ips, removed_ips, date, commit):
             if removed['ip']:
                 if removed['org']:
                     stats.unlocked[added['org']] += 1
+                else:
+                    stats.no_org += 1
             elif removed['ip_subnet']:
                 if removed['org']:
                     stats.unlocked[added['org']] += count_network_ips(removed['ip_subnet'])
+                else:
+                    stats.no_org += count_network_ips(removed['ip_subnet'])
             else:
                 raise Exception("Bad ip data: " + str(removed))
         except Exception as e:
             logger.error('{0}\t{1}\t{2}\t{3}'.format(commit, date, removed, e))
 
+    logger_info.info(str(stats))
     for org in set(stats.blocked.keys()) | set(stats.unlocked.keys()):
         session.add(Stats(date, stats.blocked[org], stats.unlocked[org], org))
 
@@ -233,4 +248,4 @@ def make_cache():
     session = Session()
     update(repo_path, session)
     make_cache()
-    
+