diff --git a/pogom/models.py b/pogom/models.py index a214d16b3e..ac81197f12 100644 --- a/pogom/models.py +++ b/pogom/models.py @@ -866,14 +866,14 @@ def get_by_cellids(cls, cellids): d = {} for sl in list(query): - key = "{},{}".format(sl['latitude'], sl['longitude']) + key = "{}".format(sl['cellid']) d[key] = sl return d @classmethod def find_in_locs(cls, loc, locs): - key = "{},{}".format(loc[0], loc[1]) + key = "{}".format(cellid(loc)) return locs[key] if key in locs else cls.new_loc(loc) # Return value of a particular scan from loc, or default dict if not found. @@ -895,8 +895,13 @@ def link_spawn_points(cls, scans, initial, spawn_points, distance, for cell, scan in scans.iteritems(): if initial[cell]['done'] and not force: continue - + # Difference in degrees at the equator for 70m is actually 0.00063 + # degrees and gets smaller the further north or south you go + deg_at_lat = 0.0007 / math.cos(math.radians(scan['loc'][0])) for sp in spawn_points: + if (abs(sp['latitude'] - scan['loc'][0]) > 0.0008 or + abs(sp['longitude'] - scan['loc'][1]) > deg_at_lat): + continue if in_radius((sp['latitude'], sp['longitude']), scan['loc'], distance): scan_spawn_point[cell + sp['id']] = { @@ -921,12 +926,38 @@ def linked_spawn_points(cls, cell): # Return list of dicts for upcoming valid band times. @classmethod - def get_cell_to_linked_spawn_points(cls, cellids): + def get_cell_to_linked_spawn_points(cls, cellids, location_change_date): + + # Get all spawnpoints from the hive's cells + sp_from_cells = (ScanSpawnPoint + .select(ScanSpawnPoint.spawnpoint) + .where(ScanSpawnPoint.scannedlocation << cellids) + .alias('spcells')) + # A new SL (new ones are created when the location changes) or + # it can be a cell from another active hive + one_sp_scan = (ScanSpawnPoint + .select(ScanSpawnPoint.spawnpoint, + fn.MAX(ScanSpawnPoint.scannedlocation).alias( + 'cellid')) + .join(sp_from_cells, on=sp_from_cells.c.spawnpoint_id + == ScanSpawnPoint.spawnpoint) + .join(cls, on=(cls.cellid == + ScanSpawnPoint.scannedlocation)) + .where(((cls.last_modified >= (location_change_date)) & + (cls.last_modified > ( + datetime.utcnow() - timedelta(minutes=60)))) | + (cls.cellid << cellids)) + .group_by(ScanSpawnPoint.spawnpoint) + .alias('maxscan')) + # As scan locations overlap,spawnpoints can belong to up to 3 locations + # This sub-query effectively assigns each SP to exactly one location. + query = (SpawnPoint - .select(SpawnPoint, cls.cellid) - .join(ScanSpawnPoint) - .join(cls) - .where(cls.cellid << cellids).dicts()) + .select(SpawnPoint, one_sp_scan.c.cellid) + .join(one_sp_scan, on=(SpawnPoint.id == + one_sp_scan.c.spawnpoint_id)) + .where(one_sp_scan.c.cellid << cellids) + .dicts()) l = list(query) ret = {} for item in l: @@ -987,8 +1018,8 @@ def get_times(cls, scan, now_date, scanned_locations): # Checks if now falls within an unfilled band for a scanned location. # Returns the updated scan location dict. @classmethod - def update_band(cls, scan): - now_date = datetime.utcnow() + def update_band(cls, scan, now_date): + scan['last_modified'] = now_date if scan['done']: @@ -1045,40 +1076,22 @@ def reset_bands(cls, scan_loc): scan_loc['band' + str(i)] = -1 @classmethod - def select_in_hex(cls, center, steps): + def select_in_hex(cls, locs): # There should be a way to delegate this to SpawnPoint.select_in_hex, # but w/e. + cells = [] + for i, e in enumerate(locs): + cells.append(cellid(e[1])) - R = 6378.1 # KM radius of the earth - hdist = ((steps * 120.0) - 50.0) / 1000.0 - n, e, s, w = hex_bounds(center, steps) - - # Get all spawns in that box. + # Get all spawns for the locations. sp = list(cls .select() - .where((cls.latitude <= n) & - (cls.latitude >= s) & - (cls.longitude >= w) & - (cls.longitude <= e)) + .where(cls.cellid << cells) .dicts()) # For each spawn work out if it is in the hex (clipping the diagonals). in_hex = [] for spawn in sp: - # Get the offset from the center of each spawn in km. - offset = [math.radians(spawn['latitude'] - center[0]) * R, - math.radians(spawn['longitude'] - center[1]) * - (R * math.cos(math.radians(center[0])))] - # Check against the 4 lines that make up the diagonals. - if (offset[1] + (offset[0] * 0.5)) > hdist: # Too far NE. - continue - if (offset[1] - (offset[0] * 0.5)) > hdist: # Too far SE - continue - if ((offset[0] * 0.5) - offset[1]) > hdist: # Too far NW - continue - if ((0 - offset[1]) - (offset[0] * 0.5)) > hdist: # Too far SW - continue - # If it gets to here it's a good spawn. in_hex.append(spawn) return in_hex @@ -1297,25 +1310,35 @@ def get_times(cls, cell, scan, now_date, scan_delay, continue endpoints = SpawnPoint.start_end(sp, scan_delay) - cls.add_if_not_scanned('spawn', l, sp, scan, endpoints[0], - endpoints[1], now_date, now_secs, sp_by_id) + cls.add_if_not_scanned('spawn', l, sp, scan, + endpoints[0], endpoints[1], now_date, + now_secs, sp_by_id) # Check to see if still searching for valid TTH. if cls.tth_found(sp): continue # Add a spawnpoint check between latest_seen and earliest_unseen. - start = sp['latest_seen'] + scan_delay + start = sp['latest_seen'] end = sp['earliest_unseen'] - cls.add_if_not_scanned( - 'TTH', l, sp, scan, start, end, now_date, now_secs, sp_by_id) + # So if the gap between start and end < 89 seconds make the gap + # 89 seconds + if ((end > start and end - start < 89) or + (start > end and (end + 3600) - start < 89)): + end = (start + 89) % 3600 + # So we move the search gap on 45 to within 45 and 89 seconds from + # the last scan. TTH appears in the last 90 seconds of the Spawn. + start = sp['latest_seen'] + 45 + + cls.add_if_not_scanned('TTH', l, sp, scan, + start, end, now_date, now_secs, sp_by_id) return l @classmethod - def add_if_not_scanned(cls, kind, l, sp, scan, start, end, now_date, - now_secs, sp_by_id): + def add_if_not_scanned(cls, kind, l, sp, scan, start, + end, now_date, now_secs, sp_by_id): # Make sure later than now_secs. while end < now_secs: start, end = start + 3600, end + 3600 @@ -1324,8 +1347,11 @@ def add_if_not_scanned(cls, kind, l, sp, scan, start, end, now_date, while start > end: start -= 3600 + while start < 0: + start, end = start + 3600, end + 3600 + last_scanned = sp_by_id[sp['id']]['last_scanned'] - if (now_date - last_scanned).total_seconds() > now_secs - start: + if ((now_date - last_scanned).total_seconds() > now_secs - start): l.append(ScannedLocation._q_init(scan, start, end, kind, sp['id'])) # Given seconds after the hour and a spawnpoint dict, return which quartile @@ -1336,7 +1362,45 @@ def get_quartile(secs, sp): 3600) / 15 / 60) @classmethod - def select_in_hex(cls, center, steps): + def select_in_hex_by_cellids(cls, cellids, location_change_date): + # Get all spawnpoints from the hive's cells + sp_from_cells = (ScanSpawnPoint + .select(ScanSpawnPoint.spawnpoint) + .where(ScanSpawnPoint.scannedlocation << cellids) + .alias('spcells')) + # Allocate a spawnpoint to one cell only, this can either be + # A new SL (new ones are created when the location changes) or + # it can be a cell from another active hive + one_sp_scan = (ScanSpawnPoint + .select(ScanSpawnPoint.spawnpoint, + fn.MAX(ScanSpawnPoint.scannedlocation).alias( + 'Max_ScannedLocation_id')) + .join(sp_from_cells, on=sp_from_cells.c.spawnpoint_id + == ScanSpawnPoint.spawnpoint) + .join(ScannedLocation, on=(ScannedLocation.cellid + == ScanSpawnPoint.scannedlocation)) + .where(((ScannedLocation.last_modified + >= (location_change_date)) & ( + ScannedLocation.last_modified > ( + datetime.utcnow() - timedelta(minutes=60)))) | + (ScannedLocation.cellid << cellids)) + .group_by(ScanSpawnPoint.spawnpoint) + .alias('maxscan')) + + query = (cls + .select(cls) + .join(one_sp_scan, + on=(one_sp_scan.c.spawnpoint_id == cls.id)) + .where(one_sp_scan.c.Max_ScannedLocation_id << cellids) + .dicts()) + + in_hex = [] + for spawn in list(query): + in_hex.append(spawn) + return in_hex + + @classmethod + def select_in_hex_by_location(cls, center, steps): R = 6378.1 # KM radius of the earth hdist = ((steps * 120.0) - 50.0) / 1000.0 n, e, s, w = hex_bounds(center, steps) @@ -1390,25 +1454,11 @@ class SpawnpointDetectionData(BaseModel): @staticmethod def set_default_earliest_unseen(sp): - sp['earliest_unseen'] = (sp['latest_seen'] + 14 * 60) % 3600 + sp['earliest_unseen'] = (sp['latest_seen'] + 15 * 60) % 3600 @classmethod def classify(cls, sp, scan_loc, now_secs, sighting=None): - # To reduce CPU usage, give an intial reading of 15 minute spawns if - # not done with initial scan of location. - if not scan_loc['done']: - sp['kind'] = 'hhhs' - if not sp['earliest_unseen']: - sp['latest_seen'] = now_secs - cls.set_default_earliest_unseen(sp) - - elif clock_between(sp['latest_seen'], now_secs, - sp['earliest_unseen']): - sp['latest_seen'] = now_secs - - return - # Get past sightings. query = list(cls.select() .where(cls.spawnpoint_id == sp['id']) @@ -1418,13 +1468,37 @@ def classify(cls, sp, scan_loc, now_secs, sighting=None): if sighting: query.append(sighting) + tth_found = False + for s in query: + if s['tth_secs'] is not None: + tth_found = True + tth_secs = (s['tth_secs'] - 1) % 3600 + + # To reduce CPU usage, give an intial reading of 15 minute spawns if + # not done with initial scan of location. + if not scan_loc['done']: + # We only want to reset a SP if it is new and not due the + # location changing (which creates new Scannedlocations) + if not tth_found: + sp['kind'] = 'hhhs' + if not sp['earliest_unseen']: + sp['latest_seen'] = now_secs + cls.set_default_earliest_unseen(sp) + + elif clock_between(sp['latest_seen'], now_secs, + sp['earliest_unseen']): + sp['latest_seen'] = now_secs + return + # Make a record of links, so we can reset earliest_unseen # if it changes. old_kind = str(sp['kind']) - # Make a sorted list of the seconds after the hour. seen_secs = sorted(map(lambda x: date_secs(x['scan_time']), query)) - + # Include and entry for the TTH if it found + if tth_found: + seen_secs.append(tth_secs) + seen_secs.sort() # Add the first seen_secs to the end as a clock wrap around. if seen_secs: seen_secs.append(seen_secs[0] + 3600) @@ -1438,7 +1512,7 @@ def classify(cls, sp, scan_loc, now_secs, sighting=None): # An hour minus the largest gap in minutes gives us the duration the # spawn was there. Round up to the nearest 15 minute interval for our # current best guess duration. - duration = (int((59 - max_gap / 60.0) / 15) + 1) * 15 + duration = (int((60 - max_gap / 60.0) / 15) + 1) * 15 # If the second largest gap is larger than 15 minutes, then there are # two gaps greater than 15 minutes. It must be a double spawn. @@ -1458,7 +1532,8 @@ def classify(cls, sp, scan_loc, now_secs, sighting=None): if sp['kind'] != 'ssss': if (not sp['earliest_unseen'] or - sp['earliest_unseen'] != sp['latest_seen']): + sp['earliest_unseen'] != sp['latest_seen'] or + not tth_found): # New latest_seen will be just before max_gap. sp['latest_seen'] = seen_secs[gap_list.index(max_gap)] @@ -1467,7 +1542,6 @@ def classify(cls, sp, scan_loc, now_secs, sighting=None): # spawn has changed, reset to latest_seen + 14 minutes. if not sp['earliest_unseen'] or sp['kind'] != old_kind: cls.set_default_earliest_unseen(sp) - return # Only ssss spawns from here below. @@ -1704,7 +1778,11 @@ def parse_map(args, map_dict, step_location, db_update_queue, wh_update_queue, # Consolidate the individual lists in each cell into two lists of Pokemon # and a list of forts. cells = map_dict['responses']['GET_MAP_OBJECTS']['map_cells'] - for cell in cells: + for i, cell in enumerate(cells): + # If we have map responses then use the time from the request + if i == 0: + now_date = datetime.utcfromtimestamp( + cell['current_timestamp_ms'] / 1000) nearby_pokemon += cell.get('nearby_pokemons', []) # Parse everything for stats (counts). Future enhancement -- we don't # necessarily need to know *how many* forts/wild/nearby were found but @@ -1714,40 +1792,22 @@ def parse_map(args, map_dict, step_location, db_update_queue, wh_update_queue, forts += cell.get('forts', []) + now_secs = date_secs(now_date) # If there are no wild or nearby Pokemon . . . if not wild_pokemon and not nearby_pokemon: # . . . and there are no gyms/pokestops then it's unusable/bad. - abandon_loc = False - if not forts: log.warning('Bad scan. Parsing found absolutely nothing.') log.info('Common causes: captchas or IP bans.') - abandon_loc = True else: # No wild or nearby Pokemon but there are forts. It's probably # a speed violation. log.warning('No nearby or wild Pokemon but there are visible gyms ' 'or pokestops. Possible speed violation.') - if not (config['parse_pokestops'] or config['parse_gyms']): - # If we're not going to parse the forts, then we'll just - # exit here. - abandon_loc = True - - if abandon_loc: - scan_loc = ScannedLocation.get_by_loc(step_location) - ScannedLocation.update_band(scan_loc) - db_update_queue.put((ScannedLocation, {0: scan_loc})) - - return { - 'count': 0, - 'gyms': gyms, - 'spawn_points': spawn_points, - 'bad_scan': True - } scan_loc = ScannedLocation.get_by_loc(step_location) done_already = scan_loc['done'] - ScannedLocation.update_band(scan_loc) + ScannedLocation.update_band(scan_loc, now_date) just_completed = not done_already and scan_loc['done'] if wild_pokemon and config['parse_pokemon']: @@ -1757,7 +1817,7 @@ def parse_map(args, map_dict, step_location, db_update_queue, wh_update_queue, # the database. query = (Pokemon .select(Pokemon.encounter_id, Pokemon.spawnpoint_id) - .where((Pokemon.disappear_time > datetime.utcnow()) & + .where((Pokemon.disappear_time >= now_date) & (Pokemon.encounter_id << encounter_ids)) .dicts()) @@ -1790,7 +1850,8 @@ def parse_map(args, map_dict, step_location, db_update_queue, wh_update_queue, d_t_secs = date_secs(datetime.utcfromtimestamp( (p['last_modified_timestamp_ms'] + p['time_till_hidden_ms']) / 1000.0)) - if sp['latest_seen'] != sp['earliest_unseen']: + if (sp['latest_seen'] != sp['earliest_unseen'] or + not sp['last_scanned']): log.info('TTH found for spawnpoint %s.', sp['id']) sighting['tth_secs'] = d_t_secs @@ -2046,30 +2107,37 @@ def parse_map(args, map_dict, step_location, db_update_queue, wh_update_queue, # Don't overwrite changes from this parse with DB version. sp = spawn_points[sp['id']] else: + # If the cell has completed, we need to classify all + # the SPs that were not picked up in the scan + if just_completed: + SpawnpointDetectionData.classify(sp, scan_loc, now_secs) + spawn_points[sp['id']] = sp if SpawnpointDetectionData.unseen(sp, now_secs): spawn_points[sp['id']] = sp endpoints = SpawnPoint.start_end(sp, args.spawn_delay) if clock_between(endpoints[0], now_secs, endpoints[1]): sp['missed_count'] += 1 spawn_points[sp['id']] = sp - log.warning('%s kind spawnpoint %s has no Pokemon %d times ' - 'in a row.', sp['kind'], sp['id'], - sp['missed_count']) - log.info('Possible causes: Still doing initial scan, super ' - 'rare double spawnpoint during') - log.info('hidden period, or Niantic has removed spawnpoint.') + log.warning('%s kind spawnpoint %s has no Pokemon %d times' + ' in a row.', + sp['kind'], sp['id'], sp['missed_count']) + log.info('Possible causes: Still doing initial scan, super' + ' rare double spawnpoint during') + log.info('hidden period, or Niantic has removed ' + 'spawnpoint.') if (not SpawnPoint.tth_found(sp) and scan_loc['done'] and - (sp['earliest_unseen'] - sp['latest_seen'] - + (now_secs - sp['latest_seen'] - args.spawn_delay) % 3600 < 60): - log.warning('Spawnpoint %s was unable to locate a TTH, with only ' - '%ss after Pokemon last seen.', sp['id'], - (sp['earliest_unseen'] - sp['latest_seen']) % 3600) + log.warning('Spawnpoint %s was unable to locate a TTH, with ' + 'only %ss after Pokemon last seen.', sp['id'], + (now_secs - sp['latest_seen']) % 3600) log.info('Restarting current 15 minute search for TTH.') if sp['id'] not in sp_id_list: SpawnpointDetectionData.classify(sp, scan_loc, now_secs) sp['latest_seen'] = (sp['latest_seen'] - 60) % 3600 - sp['earliest_unseen'] = (sp['earliest_unseen'] + 14 * 60) % 3600 + sp['earliest_unseen'] = ( + sp['earliest_unseen'] + 14 * 60) % 3600 spawn_points[sp['id']] = sp db_update_queue.put((ScannedLocation, {0: scan_loc})) @@ -2093,14 +2161,16 @@ def parse_map(args, map_dict, step_location, db_update_queue, wh_update_queue, 'count': len(wild_pokemon) + len(forts), 'gyms': gyms, 'sp_id_list': sp_id_list, - 'bad_scan': True + 'bad_scan': True, + 'scan_secs': now_secs } return { 'count': len(wild_pokemon) + len(forts), 'gyms': gyms, 'sp_id_list': sp_id_list, - 'bad_scan': False + 'bad_scan': False, + 'scan_secs': now_secs } diff --git a/pogom/schedulers.py b/pogom/schedulers.py index 3bc8133272..46535b88cf 100644 --- a/pogom/schedulers.py +++ b/pogom/schedulers.py @@ -62,7 +62,7 @@ from .transform import get_new_coords from .models import (hex_bounds, Pokemon, SpawnPoint, ScannedLocation, ScanSpawnPoint) -from .utils import now, cur_sec, cellid, date_secs, equi_rect_distance +from .utils import now, cur_sec, cellid, equi_rect_distance from .altitude import get_altitude log = logging.getLogger(__name__) @@ -139,7 +139,7 @@ def next_item(self, search_items_queue): 'abandoning location.').format(step_location[0], step_location[1]) } - return step, step_location, appears, leaves, messages + return step, step_location, appears, leaves, messages, 0 # How long to delay since last action def delay(self, *args): @@ -480,8 +480,11 @@ def __init__(self, queues, status, args): super(SpeedScan, self).__init__(queues, status, args) self.refresh_date = datetime.utcnow() - timedelta(days=1) self.next_band_date = self.refresh_date + self.location_change_date = datetime.utcnow() self.queues = [[]] + self.queue_version = 0 self.ready = False + self.empty_hive = False self.spawns_found = 0 self.spawns_missed_delay = {} self.scans_done = 0 @@ -512,12 +515,12 @@ def _locks_init(self): # On location change, empty the current queue and the locations list def location_changed(self, scan_location, db_update_queue): super(SpeedScan, self).location_changed(scan_location, db_update_queue) + self.location_change_date = datetime.utcnow() self.locations = self._generate_locations() scans = {} initial = {} all_scans = {} - for sl in ScannedLocation.select_in_hex(self.scan_location, - self.args.step_limit): + for sl in ScannedLocation.select_in_hex(self.locations): all_scans[cellid((sl['latitude'], sl['longitude']))] = sl for i, e in enumerate(self.locations): @@ -533,7 +536,7 @@ def location_changed(self, scan_location, db_update_queue): log.info('%d steps created', len(scans)) self.band_spacing = int(10 * 60 / len(scans)) self.band_status() - spawnpoints = SpawnPoint.select_in_hex( + spawnpoints = SpawnPoint.select_in_hex_by_location( self.scan_location, self.args.step_limit) if not spawnpoints: log.info('No spawnpoints in hex found in SpawnPoint table. ' + @@ -560,52 +563,27 @@ def location_changed(self, scan_location, db_update_queue): # since it didn't recognize the location in the ScannedLocation table def _generate_locations(self): - NORTH = 0 - EAST = 90 - SOUTH = 180 - WEST = 270 - # dist between column centers xdist = math.sqrt(3) * self.step_distance - ydist = 3 * (self.step_distance / 2) # dist between row centers results = [] - loc = self.scan_location results.append((loc[0], loc[1], 0)) - - # upper part + # This will loop thorugh all the rings in the hex from the centre + # moving outwards for ring in range(1, self.step_limit): - - for i in range(max(ring - 1, 1)): - if ring > 1: - loc = get_new_coords(loc, ydist, NORTH) - - loc = get_new_coords(loc, xdist / (1 + (ring > 1)), WEST) - results.append((loc[0], loc[1], 0)) - - for i in range(ring): - loc = get_new_coords(loc, ydist, NORTH) - loc = get_new_coords(loc, xdist / 2, EAST) - results.append((loc[0], loc[1], 0)) - - for i in range(ring): - loc = get_new_coords(loc, xdist, EAST) - results.append((loc[0], loc[1], 0)) - - for i in range(ring): - loc = get_new_coords(loc, ydist, SOUTH) - loc = get_new_coords(loc, xdist / 2, EAST) - results.append((loc[0], loc[1], 0)) - - for i in range(ring): - loc = get_new_coords(loc, ydist, SOUTH) - loc = get_new_coords(loc, xdist / 2, WEST) - results.append((loc[0], loc[1], 0)) - - for i in range(ring + (ring + 1 < self.step_limit)): - loc = get_new_coords(loc, xdist, WEST) - results.append((loc[0], loc[1], 0)) + for i in range(0, 6): + # Star_locs will contain the locations of the 6 vertices of + # the current ring (90,150,210,270,330 and 30 degrees from + # origin) to form a star + star_loc = get_new_coords(self.scan_location, xdist * ring, + 90 + 60*i) + for j in range(0, ring): + # Then from each point on the star, create locations + # towards the next point of star along the edge of the + # current ring + loc = get_new_coords(star_loc, xdist * (j), 210 + 60*i) + results.append((loc[0], loc[1], 0)) generated_locations = [] for step, location in enumerate(results): @@ -651,7 +629,8 @@ def get_overseer_message(self): # the first band of a scan is done def time_to_refresh_queue(self): return ((datetime.utcnow() - self.refresh_date).total_seconds() > - self.minutes * 60 or self.queues == [[]]) + self.minutes * 60 or + (self.queues == [[]] and not self.empty_hive)) # Function to empty all queues in the queues list def empty_queues(self): @@ -690,6 +669,7 @@ def schedule(self): now_date = datetime.utcnow() self.refresh_date = now_date self.refresh_ms = now_date.minute * 60 + now_date.second + self.queue_version += 1 old_q = deepcopy(self.queues[0]) queue = [] @@ -702,7 +682,8 @@ def schedule(self): # extract all spawnpoints into a dict with spawnpoint # id -> spawnpoint for easy access later cell_to_linked_spawn_points = ( - ScannedLocation.get_cell_to_linked_spawn_points(self.scans.keys())) + ScannedLocation.get_cell_to_linked_spawn_points( + self.scans.keys(), self.location_change_date)) sp_by_id = {} for sps in cell_to_linked_spawn_points.itervalues(): for sp in sps: @@ -722,6 +703,10 @@ def schedule(self): self.ready = True log.info('New queue created with %d entries in %f seconds', len(queue), (end - start)) + # Avoiding refreshing the Queue when the initial scan is complete, and + # there are no spawnpoints in the hive. + if len(queue) == 0: + self.empty_hive = True if old_q: # Enclosing in try: to avoid divide by zero exceptions from # killing overseer @@ -754,8 +739,8 @@ def schedule(self): found_percent = 100.0 good_percent = 100.0 spawns_reached = 100.0 - spawnpoints = SpawnPoint.select_in_hex( - self.scan_location, self.args.step_limit) + spawnpoints = SpawnPoint.select_in_hex_by_cellids( + self.scans.keys(), self.location_change_date) for sp in spawnpoints: if sp['missed_count'] > 5: continue @@ -924,14 +909,14 @@ def next_item(self, status): if now_date < self.next_band_date: continue - # If the start time isn't yet, don't bother looking further, - # since queue sorted by start time. - if ms < item['start']: - break - + # If we are going to get there before it starts then ignore loc = item['loc'] distance = equi_rect_distance(loc, worker_loc) secs_to_arrival = distance / self.args.kph * 3600 + secs_waited = (now_date - last_action).total_seconds() + secs_to_arrival = max(secs_to_arrival - secs_waited, 0) + if ms + secs_to_arrival < item['start']: + continue # If we can't make it there before it disappears, don't bother # trying. @@ -950,13 +935,19 @@ def next_item(self, status): score = score / (distance + .01) if score > best['score']: - best = {'score': score, 'i': i} + best = {'score': score, 'i': i, + 'secs_to_arrival': secs_to_arrival} best.update(item) prefix = 'Calc %.2f for %d scans:' % (time.time() - now_time, n) loc = best.get('loc', []) step = best.get('step', 0) + secs_to_arrival = best.get('secs_to_arrival', 0) i = best.get('i', 0) + st = best.get('start', 0) + end = best.get('end', 0) + log.debug('step {} start {} end {} secs to arrival {}'.format( + step, st, end, secs_to_arrival)) messages = { 'wait': 'Nothing to scan.', 'early': 'Early for step {}; waiting a few seconds...'.format( @@ -976,13 +967,13 @@ def next_item(self, status): except IndexError: messages['wait'] = ('Search aborting.' + ' Overseer refreshing queue.') - return -1, 0, 0, 0, messages + return -1, 0, 0, 0, messages, 0 if best['score'] == 0: if cant_reach: messages['wait'] = ('Not able to reach any scan' + ' under the speed limit.') - return -1, 0, 0, 0, messages + return -1, 0, 0, 0, messages, 0 distance = equi_rect_distance(loc, worker_loc) if (distance > @@ -1000,7 +991,12 @@ def next_item(self, status): messages['wait'] = 'Moving {}m to step {} for a {}.'.format( int(distance * 1000), step, best['kind']) - return -1, 0, 0, 0, messages + # So we wait while the worker arrives at the destination + # But we don't want to sleep too long or the item might get + # taken by another worker + if secs_to_arrival > 179 - self.args.scan_delay: + secs_to_arrival = 179 - self.args.scan_delay + return -1, 0, 0, 0, messages, max(secs_to_arrival, 0) prefix += ' Step %d,' % (step) @@ -1012,12 +1008,12 @@ def next_item(self, status): if item.get('done', False): messages['wait'] = ('Skipping step {}. Other worker already ' + 'scanned.').format(step) - return -1, 0, 0, 0, messages + return -1, 0, 0, 0, messages, 0 if not self.ready: messages['wait'] = ('Search aborting.' + ' Overseer refreshing queue.') - return -1, 0, 0, 0, messages + return -1, 0, 0, 0, messages, 0 # If a new band, set the date to wait until for the next band. if best['kind'] == 'band' and best['end'] - best['start'] > 5 * 60: @@ -1027,24 +1023,30 @@ def next_item(self, status): # Mark scanned item['done'] = 'Scanned' status['index_of_queue_item'] = i + status['queue_version'] = self.queue_version messages['search'] = 'Scanning step {} for a {}.'.format( best['step'], best['kind']) - return best['step'], best['loc'], 0, 0, messages + return best['step'], best['loc'], 0, 0, messages, 0 def task_done(self, status, parsed=False): if parsed: # Record delay between spawn time and scanning for statistics - now_secs = date_secs(datetime.utcnow()) + # This now holds the actual time of scan in seconds + scan_secs = parsed['scan_secs'] + + # It seems that the best solution is not to interfere with the + # item if the queue has been refreshed since scanning + if status['queue_version'] != self.queue_version: + log.info('Step item has changed since queue refresh') + return item = self.queues[0][status['index_of_queue_item']] - seconds_within_band = ( - int((datetime.utcnow() - self.refresh_date).total_seconds()) + - self.refresh_ms) - enforced_delay = (self.args.spawn_delay if item['kind'] == 'spawn' - else 0) - start_delay = seconds_within_band - item['start'] + enforced_delay - safety_buffer = item['end'] - seconds_within_band - + safety_buffer = item['end'] - scan_secs + start_secs = item['start'] + if item['kind'] == 'spawn': + start_secs -= self.args.spawn_delay + start_delay = (scan_secs - start_secs) % 3600 + safety_buffer = item['end'] - scan_secs if safety_buffer < 0: log.warning('Too late by %d sec for a %s at step %d', - safety_buffer, item['kind'], item['step']) @@ -1089,8 +1091,8 @@ def task_done(self, status, parsed=False): for item in self.queues[0]: if (sp_id == item.get('sp', None) and item.get('done', None) is None and - now_secs > item['start'] and - now_secs < item['end']): + scan_secs > item['start'] and + scan_secs < item['end']): item['done'] = 'Scanned' diff --git a/pogom/search.py b/pogom/search.py index a1045e3b78..9e27a95634 100644 --- a/pogom/search.py +++ b/pogom/search.py @@ -537,7 +537,15 @@ def search_overseer_thread(args, new_location_queue, pause_bit, heartb, i].get_overseer_message() # Let's update the total stats and add that info to message - update_total_stats(threadStatus, last_account_status) + # Added exception handler as dict items change + try: + update_total_stats(threadStatus, last_account_status) + except Exception as e: + log.error( + 'Update total stats had an Exception: {}.'.format( + repr(e))) + traceback.print_exc(file=sys.stdout) + time.sleep(10) threadStatus['Overseer']['message'] += '\n' + get_stats_message( threadStatus) @@ -871,9 +879,12 @@ def search_worker_thread(args, account_queue, account_failures, break # Grab the next thing to search (when available). - step, step_location, appears, leaves, messages = ( + step, step_location, appears, leaves, messages, wait = ( scheduler.next_item(status)) status['message'] = messages['wait'] + # The next_item will return the value telling us how long + # to sleep. This way the status can be updated + time.sleep(wait) # Using step as a flag for no valid next location returned. if step == -1: @@ -1119,7 +1130,7 @@ def search_worker_thread(args, account_queue, account_failures, time.strftime( '%H:%M:%S', time.localtime(time.time() + args.scan_delay))) - + log.info(status['message']) time.sleep(delay) # Catch any process exceptions, log them, and continue the thread.