Navigation Menu

Skip to content

Commit

Permalink
no more tabs
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexeyqu
alexeyqu committed Jun 5, 2018
1 parent b03ff78 commit 59980e0
Show file tree
Hide file tree
Showing 2 changed files with 101 additions and 101 deletions.
178 changes: 89 additions & 89 deletions ip_selector.py
View file
Expand Up @@ -16,86 +16,86 @@


class Org(Enum):
FSKN = 'ФСКН'
RPN = 'Роспотребнадзор'
RKN = 'Роскомнадзор'
COURT = 'суд'
GP = 'Генпрокуратура'
MGCOURT = 'Мосгорсуд'
FNS = 'ФНС'
MVD = 'МВД'
MKS = 'Минкомсвязь'
# CWD = 'Валежник'
FSKN = 'ФСКН'
RPN = 'Роспотребнадзор'
RKN = 'Роскомнадзор'
COURT = 'суд'
GP = 'Генпрокуратура'
MGCOURT = 'Мосгорсуд'
FNS = 'ФНС'
MVD = 'МВД'
MKS = 'Минкомсвязь'
# CWD = 'Валежник'


def where_clause(orgs, ts_low, ts_high, time, group_by='latitude, longitude'):
#TODO: parameterize with ?
time_field = time
query = ' where {2} is not null and {2} >= \'{0}\' and {2} <= \'{1}\''.format(ts_low, ts_high, time_field)
query += ' and org in (\'' + str('\', \''.join([org.value for org in orgs])) + '\')'
return query + ' group by {}'.format(group_by)
#TODO: parameterize with ?
time_field = time
query = ' where {2} is not null and {2} >= \'{0}\' and {2} <= \'{1}\''.format(ts_low, ts_high, time_field)
query += ' and org in (\'' + str('\', \''.join([org.value for org in orgs])) + '\')'
return query + ' group by {}'.format(group_by)

def filter_ip(ip_dict, subnet_dict):
top_level_ip = {}
networks_binary = []
# get all networks in binary format
for _id, subnet in subnet_dict.items():
network = ipaddress.ip_network(subnet, strict=True)
networks_binary.append((get_bin_ip(ipaddress.ip_address(network.network_address)).rstrip('0'), _id, network))
networks_binary.sort(key=lambda x:x[0])
# get top networks (sorted lexicographically)
networks_top = []
current_top_addr, current_top_network = None, None
for addr, _id, network in networks_binary:
if current_top_addr is None or not addr.startswith(current_top_addr):
current_top_addr, current_top_network = addr, network
networks_top.append((current_top_addr, current_top_network))
# top_level_id.append(_id)
# get top-level individual ips
top_level_set = set()
for _id, ip in ip_dict.items():
bin_addr = get_bin_ip(ipaddress.ip_address(ip)).rstrip('0')
found = False
for addr, network in networks_top:
if bin_addr.startswith(addr):
found = True
if not found and ip not in top_level_set:
top_level_ip[_id] = ip
top_level_set.add(ip)
return top_level_ip
top_level_ip = {}
networks_binary = []
# get all networks in binary format
for _id, subnet in subnet_dict.items():
network = ipaddress.ip_network(subnet, strict=True)
networks_binary.append((get_bin_ip(ipaddress.ip_address(network.network_address)).rstrip('0'), _id, network))
networks_binary.sort(key=lambda x:x[0])
# get top networks (sorted lexicographically)
networks_top = []
current_top_addr, current_top_network = None, None
for addr, _id, network in networks_binary:
if current_top_addr is None or not addr.startswith(current_top_addr):
current_top_addr, current_top_network = addr, network
networks_top.append((current_top_addr, current_top_network))
# top_level_id.append(_id)
# get top-level individual ips
top_level_set = set()
for _id, ip in ip_dict.items():
bin_addr = get_bin_ip(ipaddress.ip_address(ip)).rstrip('0')
found = False
for addr, network in networks_top:
if bin_addr.startswith(addr):
found = True
if not found and ip not in top_level_set:
top_level_ip[_id] = ip
top_level_set.add(ip)
return top_level_ip


def select_ip(orgs=[org for org in Org], ts_low=min_date, ts_high=max_date, use_cache=True, only_locked = False):
# sorry about that..
if use_cache and len(orgs) == len(Org) and ts_low == min_date and ts_high == max_date and not only_locked and os.path.isfile(full_geo_cache):
with open(full_geo_cache, 'rb') as cache:
try:
data = pickle.load(cache)
return data
except:
pass
query = 'select latitude, longitude, sum(2 << (31 - length(prefix))), 1 as type, max(include_time) as time from blocked_ip'
query += ' join geo_prefix on (prefix between (ip_bin || \'0\') and (ip_bin || \'1\')) or (prefix = ip_bin)'
query += ' join block_geo on (block_geo.id = geo_id)'
query += where_clause(orgs, ts_low, ts_high, 'include_time')
query += ' union '
query += 'select latitude, longitude, count(*), 1 as type, max(include_time) as time from blocked_ip'
query += ' join block_geo on (block_id = blocked_ip.id) and (ip_subnet is null)'
query += where_clause(orgs, ts_low, ts_high, 'include_time')
if not only_locked:
query += ' union '
query += 'select latitude, longitude, sum(2 << (31 - length(prefix))), 0 as type, max(exclude_time) as time from blocked_ip'
query += ' join geo_prefix on (prefix between (ip_bin || \'0\') and (ip_bin || \'1\')) or (prefix = ip_bin)'
query += ' join block_geo on (block_geo.id = geo_id)'
query += where_clause(orgs, ts_low, ts_high, 'exclude_time')
query += ' union '
query += 'select latitude, longitude, count(*), 0 as type, max(exclude_time) as time from blocked_ip'
query += ' join block_geo on (block_id = blocked_ip.id) and (ip_subnet is null)'
query += where_clause(orgs, ts_low, ts_high, 'exclude_time')
query += ' order by time, type desc'
data = engine.execute(query).fetchall()
return data
# sorry about that..
if use_cache and len(orgs) == len(Org) and ts_low == min_date and ts_high == max_date and not only_locked and os.path.isfile(full_geo_cache):
with open(full_geo_cache, 'rb') as cache:
try:
data = pickle.load(cache)
return data
except:
pass
query = 'select latitude, longitude, sum(2 << (31 - length(prefix))), 1 as type, max(include_time) as time from blocked_ip'
query += ' join geo_prefix on (prefix between (ip_bin || \'0\') and (ip_bin || \'1\')) or (prefix = ip_bin)'
query += ' join block_geo on (block_geo.id = geo_id)'
query += where_clause(orgs, ts_low, ts_high, 'include_time')
query += ' union '
query += 'select latitude, longitude, count(*), 1 as type, max(include_time) as time from blocked_ip'
query += ' join block_geo on (block_id = blocked_ip.id) and (ip_subnet is null)'
query += where_clause(orgs, ts_low, ts_high, 'include_time')
if not only_locked:
query += ' union '
query += 'select latitude, longitude, sum(2 << (31 - length(prefix))), 0 as type, max(exclude_time) as time from blocked_ip'
query += ' join geo_prefix on (prefix between (ip_bin || \'0\') and (ip_bin || \'1\')) or (prefix = ip_bin)'
query += ' join block_geo on (block_geo.id = geo_id)'
query += where_clause(orgs, ts_low, ts_high, 'exclude_time')
query += ' union '
query += 'select latitude, longitude, count(*), 0 as type, max(exclude_time) as time from blocked_ip'
query += ' join block_geo on (block_id = blocked_ip.id) and (ip_subnet is null)'
query += where_clause(orgs, ts_low, ts_high, 'exclude_time')
query += ' order by time, type desc'
data = engine.execute(query).fetchall()
return data


def select_stats(orgs=[org for org in Org], ts_low=min_date, ts_high=max_date, only_locked=False):
Expand Down Expand Up @@ -128,29 +128,29 @@ def extend_stats(stats):
start_ts = make_ts(data[0]['date'])
stats = [('Заблокировано', '#FF0000', start_ts, [item['blocked'] for item in data])]
if not only_locked:
stats.append(('Разблокировано', '#00FF00', start_ts, [item['unlocked'] for item in data]))
stats.append(('Разблокировано', '#00FF00', start_ts, [item['unlocked'] for item in data]))
return stats
else:
return []


def smart_print(orgs):
pass
#print('{0} : {1}'.format(orgs, len(select_ip(orgs))))
pass
#print('{0} : {1}'.format(orgs, len(select_ip(orgs))))


if __name__ == '__main__':
smart_print([])
smart_print([Org.FSKN])
smart_print([Org.RPN])
smart_print([Org.RKN])
smart_print([Org.COURT])
smart_print([Org.GP])
smart_print([Org.MGCOURT])
smart_print([Org.FNS])
smart_print([Org.MVD])
smart_print([Org.MKS])
res = select_ip()
for row in res:
print(row)
smart_print([])
smart_print([Org.FSKN])
smart_print([Org.RPN])
smart_print([Org.RKN])
smart_print([Org.COURT])
smart_print([Org.GP])
smart_print([Org.MGCOURT])
smart_print([Org.FNS])
smart_print([Org.MVD])
smart_print([Org.MKS])
res = select_ip()
for row in res:
print(row)
24 changes: 12 additions & 12 deletions maxmind_client.py
View file
Expand Up @@ -11,15 +11,15 @@


def get_locations_for_ip(addr, try_requests=True):
try:
loc = reader.city(addr).location
return [{'latitude': loc.latitude,
'longitude': loc.longitude,
'covered_percentage': 100,
'prefixes': [addr]}]
except Exception as e:
if try_requests:
time.sleep(0.16) #API limitations
return requests.get('https://stat.ripe.net/data/geoloc/data.json?resource=' + addr).json()['data']['locations']
else:
return None
try:
loc = reader.city(addr).location
return [{'latitude': loc.latitude,
'longitude': loc.longitude,
'covered_percentage': 100,
'prefixes': [addr]}]
except Exception as e:
if try_requests:
time.sleep(0.16) #API limitations
return requests.get('https://stat.ripe.net/data/geoloc/data.json?resource=' + addr).json()['data']['locations']
else:
return None

0 comments on commit 59980e0

Please sign in to comment.