diff --git a/docs/first-run/commandline.md b/docs/first-run/commandline.md index f9c83d1be7..e16d5ce16b 100644 --- a/docs/first-run/commandline.md +++ b/docs/first-run/commandline.md @@ -18,18 +18,21 @@ [-ld LOGIN_DELAY] [-lr LOGIN_RETRIES] [-mf MAX_FAILURES] [-me MAX_EMPTY] [-bsr BAD_SCAN_RETRY] [-msl MIN_SECONDS_LEFT] [-dc] [-H HOST] [-P PORT] - [-L LOCALE] [-c] [-m MOCK] [-ns] [-os] [-sc] [-nfl] -k - GMAPS_KEY [--skip-empty] [-C] [-cd] [-np] [-ng] [-nr] + [-L LOCALE] [-c] [-m MOCK] [-ns] [-os] [-sc] [-nfl] + -k GMAPS_KEY [--skip-empty] [-C] [-cd] [-np] [-ng] [-nr] [-nk] [-ss] [-ssct SS_CLUSTER_TIME] [-speed] [-spin] [-ams ACCOUNT_MAX_SPINS] [-kph KPH] [-hkph HLVL_KPH] - [-ldur LURE_DURATION] [-pd PURGE_DATA] [-px PROXY] [-pxsc] + [-ldur LURE_DURATION] [-px PROXY] [-pxsc] [-pxt PROXY_TEST_TIMEOUT] [-pxre PROXY_TEST_RETRIES] [-pxbf PROXY_TEST_BACKOFF_FACTOR] [-pxc PROXY_TEST_CONCURRENCY] [-pxd PROXY_DISPLAY] [-pxf PROXY_FILE] [-pxr PROXY_REFRESH] [-pxo PROXY_ROTATION] --db-name DB_NAME --db-user DB_USER --db-pass DB_PASS [--db-host DB_HOST] [--db-port DB_PORT] - [--db-threads DB_THREADS] [-wh WEBHOOKS] [-gi] [-DC] + [--db-threads DB_THREADS] [-DC] [-DCw DB_CLEANUP_WORKER] + [-DCp DB_CLEANUP_POKEMON] [-DCg DB_CLEANUP_GYM] + [-DCs DB_CLEANUP_SPAWNPOINT] [-DCf DB_CLEANUP_FORTS] + [-wh WEBHOOKS] [-gi] [--wh-types {pokemon,gym,raid,egg,tth,gym-info,pokestop,lure,captcha}] [--wh-threads WH_THREADS] [-whc WH_CONCURRENCY] [-whr WH_RETRIES] [-whct WH_CONNECT_TIMEOUT] @@ -37,14 +40,14 @@ [-whlfu WH_LFU_SIZE] [-whfi WH_FRAME_INTERVAL] [--ssl-certificate SSL_CERTIFICATE] [--ssl-privatekey SSL_PRIVATEKEY] [-ps [logs]] - [-slt STATS_LOG_TIMER] [-sn STATUS_NAME] - [-spp STATUS_PAGE_PASSWORD] [-hk HASH_KEY] [-novc] - [-vci VERSION_CHECK_INTERVAL] [-odt ON_DEMAND_TIMEOUT] - [--disable-blacklist] [-tp TRUSTED_PROXIES] - [--api-version API_VERSION] [--no-file-logs] - [--log-path LOG_PATH] [--dump] [-exg] + [-slt STATS_LOG_TIMER] [-sn STATUS_NAME] [-hk HASH_KEY] + [-novc] [-vci VERSION_CHECK_INTERVAL] + [-odt ON_DEMAND_TIMEOUT] [--disable-blacklist] + [-tp TRUSTED_PROXIES] [--api-version API_VERSION] + [--no-file-logs] [--log-path LOG_PATH] [--dump] [-exg] [-v | --verbosity VERBOSE] [-Rh RARITY_HOURS] - [-Rf RARITY_UPDATE_FREQUENCY] + [-Rf RARITY_UPDATE_FREQUENCY] [-SPp STATUS_PAGE_PASSWORD] + [-SPf STATUS_PAGE_FILTER] Args that start with '--' (eg. -a) can also be set in a config file (config/config.ini or specified via -cf or -scf). The recognized syntax @@ -265,20 +268,16 @@ environment variables which override config file values which override defaults. -ams ACCOUNT_MAX_SPINS, --account-max-spins ACCOUNT_MAX_SPINS Maximum number of Pokestop spins per hour. [env var: POGOMAP_ACCOUNT_MAX_SPINS] - -kph KPH, --kph KPH Set a maximum speed in km/hour for scanner movement. 0 - to disable. Default: 35. [env var: POGOMAP_KPH] + -kph KPH, --kph KPH Set a maximum speed in km/hour for scanner movement. + Default: 35, 0 to disable. [env var: POGOMAP_KPH] -hkph HLVL_KPH, --hlvl-kph HLVL_KPH Set a maximum speed in km/hour for scanner movement, - for high-level (L30) accounts. 0 to disable. Default: - 25. [env var: POGOMAP_HLVL_KPH] + for high-level (L30) accounts. Default: 25, 0 to + disable. [env var: POGOMAP_HLVL_KPH] -ldur LURE_DURATION, --lure-duration LURE_DURATION Change duration for lures set on pokestops. This is useful for events that extend lure duration. [env var: POGOMAP_LURE_DURATION] - -pd PURGE_DATA, --purge-data PURGE_DATA - Clear Pokemon from database this many hours after they - disappear (0 to disable). [env var: - POGOMAP_PURGE_DATA] -px PROXY, --proxy PROXY Proxy url (e.g. socks5://127.0.0.1:9050) [env var: POGOMAP_PROXY] @@ -318,7 +317,6 @@ environment variables which override config file values which override defaults. var: POGOMAP_WEBHOOK] -gi, --gym-info Get all details about gyms (causes an additional API hit for every gym). [env var: POGOMAP_GYM_INFO] - -DC, --enable-clean Enable DB cleaner. [env var: POGOMAP_ENABLE_CLEAN] --wh-types {pokemon,gym,raid,egg,tth,gym-info,pokestop,lure,captcha} Defines the type of messages to send to webhooks. [env var: POGOMAP_WH_TYPES] @@ -365,9 +363,6 @@ environment variables which override config file values which override defaults. -sn STATUS_NAME, --status-name STATUS_NAME Enable status page database update using STATUS_NAME as main worker name. [env var: POGOMAP_STATUS_NAME] - -spp STATUS_PAGE_PASSWORD, --status-page-password STATUS_PAGE_PASSWORD - Set the status page password. [env var: - POGOMAP_STATUS_PAGE_PASSWORD] -hk HASH_KEY, --hash-key HASH_KEY Key for hash server [env var: POGOMAP_HASH_KEY] -novc, --no-version-check @@ -396,7 +391,7 @@ environment variables which override config file values which override defaults. --dump Dump censored debug info about the environment and auto-upload to hastebin.com. [env var: POGOMAP_DUMP] -exg, --ex-gyms Fetch OSM parks within geofence and flag gyms that are - candidates for ex raids. Only required once per area. + candidates for EX raids. Only required once per area. [env var: POGOMAP_EX_GYMS] -v Show debug messages from RocketMap and pgoapi. Can be repeated up to 3 times. @@ -404,23 +399,56 @@ environment variables which override config file values which override defaults. var: POGOMAP_VERBOSITY] Database: - --db-name DB_NAME Name of the database to be used. [env var: - POGOMAP_DB_NAME] + --db-name DB_NAME Name of the database to be used. + [env var: POGOMAP_DB_NAME] --db-user DB_USER Username for the database. [env var: POGOMAP_DB_USER] --db-pass DB_PASS Password for the database. [env var: POGOMAP_DB_PASS] - --db-host DB_HOST IP or hostname for the database. [env var: - POGOMAP_DB_HOST] + --db-host DB_HOST IP or hostname for the database. + [env var: POGOMAP_DB_HOST] --db-port DB_PORT Port for the database. [env var: POGOMAP_DB_PORT] --db-threads DB_THREADS Number of db threads; increase if the db queue falls behind. [env var: POGOMAP_DB_THREADS] + Database Cleanup: + -DC, --db-cleanup Enable regular database cleanup thread. + [env var: POGOMAP_DB_CLEANUP] + -DCw DB_CLEANUP_WORKER, --db-cleanup-worker DB_CLEANUP_WORKER + Clear worker status from database after X minutes of + inactivity. Default: 30, 0 to disable. + [env var: POGOMAP_DB_CLEANUP_WORKER] + -DCp DB_CLEANUP_POKEMON, --db-cleanup-pokemon DB_CLEANUP_POKEMON + Clear pokemon from database X hours after they + disappeared. Default: 0, 0 to disable. + [env var: POGOMAP_DB_CLEANUP_POKEMON] + -DCg DB_CLEANUP_GYM, --db-cleanup-gym DB_CLEANUP_GYM + Clear gym details from database X hours after last gym + scan. Default: 8, 0 to disable. + [env var: POGOMAP_DB_CLEANUP_GYM] + -DCs DB_CLEANUP_SPAWNPOINT, --db-cleanup-spawnpoint DB_CLEANUP_SPAWNPOINT + Clear spawnpoint from database X hours after last + valid scan. Default: 720, 0 to disable. + [env var: POGOMAP_DB_CLEANUP_SPAWNPOINT] + -DCf DB_CLEANUP_FORTS, --db-cleanup-forts DB_CLEANUP_FORTS + Clear gyms and pokestops from database X days after + last valid scan. Default: 0, 0 to disable. + [env var: POGOMAP_DB_CLEANUP_FORTS] + Dynamic Rarity: -Rh RARITY_HOURS, --rarity-hours RARITY_HOURS Number of hours of Pokemon data to use to calculate - dynamic rarity. Decimals allowed. Default: 48. 0 to + dynamic rarity. Decimals allowed. Default: 48, 0 to use all data. [env var: POGOMAP_RARITY_HOURS] -Rf RARITY_UPDATE_FREQUENCY, --rarity-update-frequency RARITY_UPDATE_FREQUENCY How often (in minutes) the dynamic rarity should be - updated. Decimals allowed. Default: 0. 0 to disable. + updated. Decimals allowed. Default: 0, 0 to disable. [env var: POGOMAP_RARITY_UPDATE_FREQUENCY] + + Status Page: + -SPp STATUS_PAGE_PASSWORD, --status-page-password STATUS_PAGE_PASSWORD + Set the status page password. + [env var: POGOMAP_STATUS_PAGE_PASSWORD] + -SPf STATUS_PAGE_FILTER, --status-page-filter STATUS_PAGE_FILTER + Filter worker status that are inactive for X minutes. + Default: 30, 0 to disable. + [env var: POGOMAP_STATUS_PAGE_FILTER] \ No newline at end of file diff --git a/pogom/app.py b/pogom/app.py index 9f6dc60b89..b86c66087a 100644 --- a/pogom/app.py +++ b/pogom/app.py @@ -425,8 +425,14 @@ def raw_data(self): d['error'] = 'Access denied' elif (request.args.get('password', None) == args.status_page_password): - d['main_workers'] = MainWorker.get_all() - d['workers'] = WorkerStatus.get_all() + max_status_age = args.status_page_filter + if max_status_age > 0: + d['main_workers'] = MainWorker.get_recent(max_status_age) + d['workers'] = WorkerStatus.get_recent(max_status_age) + else: + d['main_workers'] = MainWorker.get_all() + d['workers'] = WorkerStatus.get_all() + return jsonify(d) def loc(self): @@ -551,8 +557,13 @@ def post_status(self): if request.form.get('password', None) == args.status_page_password: d['login'] = 'ok' - d['main_workers'] = MainWorker.get_all() - d['workers'] = WorkerStatus.get_all() + max_status_age = args.status_page_filter + if max_status_age > 0: + d['main_workers'] = MainWorker.get_recent(max_status_age) + d['workers'] = WorkerStatus.get_recent(max_status_age) + else: + d['main_workers'] = MainWorker.get_all() + d['workers'] = WorkerStatus.get_all() d['hashkeys'] = HashKeys.get_obfuscated_keys() else: d['login'] = 'failed' diff --git a/pogom/models.py b/pogom/models.py index 7c461d8e9e..dabb058016 100644 --- a/pogom/models.py +++ b/pogom/models.py @@ -1092,19 +1092,41 @@ class MainWorker(BaseModel): elapsed = IntegerField(default=0) @staticmethod - def get_account_stats(): + def get_account_stats(age_minutes=30): + stats = {'working': 0, 'captcha': 0, 'failed': 0} + timeout = datetime.utcnow() - timedelta(minutes=age_minutes) with MainWorker.database().execution_context(): account_stats = (MainWorker .select(fn.SUM(MainWorker.accounts_working), fn.SUM(MainWorker.accounts_captcha), fn.SUM(MainWorker.accounts_failed)) + .where(MainWorker.last_modified >= timeout) .scalar(as_tuple=True)) - dict = {'working': 0, 'captcha': 0, 'failed': 0} - if account_stats[0] is not None: - dict = {'working': int(account_stats[0]), + if account_stats[0] is not None: + stats.update({ + 'working': int(account_stats[0]), 'captcha': int(account_stats[1]), - 'failed': int(account_stats[2])} - return dict + 'failed': int(account_stats[2]) + }) + return stats + + @staticmethod + def get_recent(age_minutes=30): + status = [] + timeout = datetime.utcnow() - timedelta(minutes=age_minutes) + try: + with MainWorker.database().execution_context(): + query = (MainWorker + .select() + .where(MainWorker.last_modified >= timeout) + .order_by(MainWorker.worker_name.asc()) + .dicts()) + + status = [dbmw for dbmw in query] + except Exception as e: + log.exception('Failed to retrieve main worker status: %s.', e) + + return status class WorkerStatus(LatLongModel): @@ -1139,18 +1161,21 @@ def db_format(status, name='status_worker_db'): 'longitude': status.get('longitude', None)} @staticmethod - def get_recent(): + def get_recent(age_minutes=30): status = [] - with WorkerStatus.database().execution_context(): - query = (WorkerStatus - .select() - .where((WorkerStatus.last_modified >= - (datetime.utcnow() - timedelta(minutes=5)))) - .order_by(WorkerStatus.username) - .dicts()) + timeout = datetime.utcnow() - timedelta(minutes=age_minutes) + try: + with WorkerStatus.database().execution_context(): + query = (WorkerStatus + .select() + .where(WorkerStatus.last_modified >= timeout) + .order_by(WorkerStatus.username.asc()) + .dicts()) + + status = [dbws for dbws in query] + except Exception as e: + log.exception('Failed to retrieve worker status: %s.', e) - for s in query: - status.append(s) return status @staticmethod @@ -1760,17 +1785,18 @@ def get_valid(limit=15): .select() .where(Token.last_updated > valid_time) .order_by(Token.last_updated.asc()) - .limit(limit)) + .limit(limit) + .dicts()) for t in query: - token_ids.append(t.id) - tokens.append(t.token) + token_ids.append(t['id']) + tokens.append(t['token']) if tokens: - log.debug('Retrived Token IDs: {}'.format(token_ids)) - result = DeleteQuery(Token).where( - Token.id << token_ids).execute() - log.debug('Deleted {} tokens.'.format(result)) + log.debug('Retrieved Token IDs: %s.', token_ids) + query = DeleteQuery(Token).where(Token.id << token_ids) + rows = query.execute() + log.debug('Claimed and removed %d captcha tokens.', rows) except OperationalError as e: - log.error('Failed captcha token transactional query: {}'.format(e)) + log.exception('Failed captcha token transactional query: %s.', e) return tokens @@ -1783,36 +1809,31 @@ class HashKeys(BaseModel): expires = DateTimeField(null=True) last_updated = DateTimeField(default=datetime.utcnow) - @staticmethod - def get_by_key(key): - query = (HashKeys - .select() - .where(HashKeys.key == key) - .dicts()) - - return query[0] if query else { - 'maximum': 0, - 'remaining': 0, - 'peak': 0, - 'expires': None, - 'last_updated': None - } - + # Obfuscate hashing keys before sending them to the front-end. @staticmethod def get_obfuscated_keys(): - # Obfuscate hashing keys before we sent them to the front-end. hashkeys = HashKeys.get_all() for i, s in enumerate(hashkeys): hashkeys[i]['key'] = s['key'][:-9] + '*'*9 return hashkeys + # Retrieve stored 'peak' value from recently used hashing keys. @staticmethod - # Retrieve the last stored 'peak' value for each hashing key. - def getStoredPeak(key): - with Token.database().execution_context(): - query = HashKeys.select(HashKeys.peak).where(HashKeys.key == key) - result = query[0].peak if query else 0 - return result + def get_stored_peaks(): + hashkeys = {} + try: + with HashKeys.database().execution_context(): + query = (HashKeys + .select(HashKeys.key, HashKeys.peak) + .where(HashKeys.last_updated > + (datetime.utcnow() - timedelta(minutes=30))) + .dicts()) + for dbhk in query: + hashkeys[dbhk['key']] = dbhk['peak'] + except OperationalError as e: + log.exception('Failed to get hashing keys stored peaks: %s.', e) + + return hashkeys def hex_bounds(center, steps=None, radius=None): @@ -2652,61 +2673,287 @@ def db_updater(q, db): def clean_db_loop(args): + # Run regular database cleanup once every minute. + regular_cleanup_secs = 60 + # Run full database cleanup once every 10 minutes. + full_cleanup_timer = default_timer() + full_cleanup_secs = 600 while True: try: - with MainWorker.database().execution_context(): - query = (MainWorker - .delete() - .where((MainWorker.last_modified < - (datetime.utcnow() - timedelta(minutes=30))))) - query.execute() + db_cleanup_regular() - query = (WorkerStatus - .delete() - .where((WorkerStatus.last_modified < - (datetime.utcnow() - timedelta(minutes=30))))) - query.execute() - - # Remove active modifier from expired lured pokestops. - query = (Pokestop.update( - lure_expiration=None, active_fort_modifier=None).where( - Pokestop.lure_expiration < datetime.utcnow())) - query.execute() - - # Remove old (unusable) captcha tokens - query = (Token - .delete() - .where((Token.last_updated < - (datetime.utcnow() - timedelta(minutes=2))))) - query.execute() + # Remove old worker status entries. + if args.db_cleanup_worker > 0: + db_cleanup_worker_status(args.db_cleanup_worker) - # Remove expired HashKeys - query = (HashKeys - .delete() - .where(HashKeys.expires < - (datetime.now() - timedelta(days=1)))) - query.execute() - - # If desired, clear old Pokemon spawns. - if args.purge_data > 0: - log.info("Beginning purge of old Pokemon spawns.") - start = datetime.utcnow() - query = (Pokemon - .delete() - .where((Pokemon.disappear_time < - (datetime.utcnow() - - timedelta(hours=args.purge_data))))) - rows = query.execute() - end = datetime.utcnow() - diff = end - start - log.info("Completed purge of old Pokemon spawns. " - "%i deleted in %f seconds.", - rows, diff.total_seconds()) - - log.info('Regular database cleaning complete.') - time.sleep(60) + # Check if it's time to run full database cleanup. + now = default_timer() + if now - full_cleanup_timer > full_cleanup_secs: + # Remove old pokemon spawns. + if args.db_cleanup_pokemon > 0: + db_clean_pokemons(args.db_cleanup_pokemon) + + # Remove old gym data. + if args.db_cleanup_gym > 0: + db_clean_gyms(args.db_cleanup_gym) + + # Remove old and extinct spawnpoint data. + if args.db_cleanup_spawnpoint > 0: + db_clean_spawnpoints(args.db_cleanup_spawnpoint) + + # Remove old pokestop and gym locations. + if args.db_cleanup_forts > 0: + db_clean_forts(args.db_cleanup_forts) + + log.info('Full database cleanup completed.') + full_cleanup_timer = now + + time.sleep(regular_cleanup_secs) except Exception as e: - log.exception('Exception in clean_db_loop: %s', repr(e)) + log.exception('Database cleanup failed: %s.', e) + + +def db_cleanup_regular(): + log.debug('Regular database cleanup started.') + start_timer = default_timer() + + now = datetime.utcnow() + # http://docs.peewee-orm.com/en/latest/peewee/database.html#advanced-connection-management + # When using an execution context, a separate connection from the pool + # will be used inside the wrapped block and a transaction will be started. + with Token.database().execution_context(): + # Remove unusable captcha tokens. + query = (Token + .delete() + .where(Token.last_updated < now - timedelta(seconds=120))) + query.execute() + + # Remove active modifier from expired lured pokestops. + query = (Pokestop + .update(lure_expiration=None, active_fort_modifier=None) + .where(Pokestop.lure_expiration < now)) + query.execute() + + # Remove expired or inactive hashing keys. + query = (HashKeys + .delete() + .where((HashKeys.expires < now - timedelta(days=1)) | + (HashKeys.last_updated < now - timedelta(days=7)))) + query.execute() + + time_diff = default_timer() - start_timer + log.debug('Completed regular cleanup in %.6f seconds.', time_diff) + + +def db_cleanup_worker_status(age_minutes): + log.debug('Beginning cleanup of old worker status.') + start_timer = default_timer() + + worker_status_timeout = datetime.utcnow() - timedelta(minutes=age_minutes) + + with MainWorker.database().execution_context(): + # Remove status information from inactive instances. + query = (MainWorker + .delete() + .where(MainWorker.last_modified < worker_status_timeout)) + query.execute() + + # Remove worker status information that are inactive. + query = (WorkerStatus + .delete() + .where(MainWorker.last_modified < worker_status_timeout)) + query.execute() + + time_diff = default_timer() - start_timer + log.debug('Completed cleanup of old worker status in %.6f seconds.', + time_diff) + + +def db_clean_pokemons(age_hours): + log.debug('Beginning cleanup of old pokemon spawns.') + start_timer = default_timer() + + pokemon_timeout = datetime.utcnow() - timedelta(hours=age_hours) + + with Pokemon.database().execution_context(): + query = (Pokemon + .delete() + .where(Pokemon.disappear_time < pokemon_timeout)) + rows = query.execute() + log.debug('Deleted %d old Pokemon entries.', rows) + + time_diff = default_timer() - start_timer + log.debug('Completed cleanup of old pokemon spawns in %.6f seconds.', + time_diff) + + +def db_clean_gyms(age_hours, gyms_age_days=30): + log.debug('Beginning cleanup of old gym data.') + start_timer = default_timer() + + gym_info_timeout = datetime.utcnow() - timedelta(hours=age_hours) + + with Gym.database().execution_context(): + # Remove old GymDetails entries. + query = (GymDetails + .delete() + .where(GymDetails.last_scanned < gym_info_timeout)) + rows = query.execute() + log.debug('Deleted %d old GymDetails entries.', rows) + + # Remove old Raid entries. + query = (Raid + .delete() + .where(Raid.end < gym_info_timeout)) + rows = query.execute() + log.debug('Deleted %d old Raid entries.', rows) + + # Remove old GymMember entries. + query = (GymMember + .delete() + .where(GymMember.last_scanned < gym_info_timeout)) + rows = query.execute() + log.debug('Deleted %d old GymMember entries.', rows) + + # Remove old GymPokemon entries. + query = (GymPokemon + .delete() + .where(GymPokemon.last_seen < gym_info_timeout)) + rows = query.execute() + log.debug('Deleted %d old GymPokemon entries.', rows) + + time_diff = default_timer() - start_timer + log.debug('Completed cleanup of old gym data in %.6f seconds.', + time_diff) + + +def db_clean_spawnpoints(age_hours, missed=5): + log.debug('Beginning cleanup of old spawnpoint data.') + start_timer = default_timer() + # Maximum number of variables to include in a single query. + step = 500 + + spawnpoint_timeout = datetime.utcnow() - timedelta(hours=age_hours) + + with SpawnPoint.database().execution_context(): + # Select old SpawnPoint entries. + query = (SpawnPoint + .select(SpawnPoint.id) + .where((SpawnPoint.last_scanned < spawnpoint_timeout) & + (SpawnPoint.missed_count > missed)) + .dicts()) + old_sp = [(sp['id']) for sp in query] + + num_records = len(old_sp) + log.debug('Found %d old SpawnPoint entries.', num_records) + + # Remove SpawnpointDetectionData entries associated to old spawnpoints. + num_rows = 0 + for i in range(0, num_records, step): + query = (SpawnpointDetectionData + .delete() + .where((SpawnpointDetectionData.spawnpoint_id << + old_sp[i:min(i + step, num_records)]))) + num_rows += query.execute() + + # Remove old SpawnPointDetectionData entries. + query = (SpawnpointDetectionData + .delete() + .where((SpawnpointDetectionData.scan_time < + spawnpoint_timeout))) + num_rows += query.execute() + log.debug('Deleted %d old SpawnpointDetectionData entries.', num_rows) + + # Select ScannedLocation entries associated to old spawnpoints. + sl_delete = set() + for i in range(0, num_records, step): + query = (ScanSpawnPoint + .select() + .where((ScanSpawnPoint.spawnpoint << + old_sp[i:min(i + step, num_records)])) + .dicts()) + for sp in query: + sl_delete.add(sp['scannedlocation']) + log.debug('Found %d ScannedLocation entries from old spawnpoints.', + len(sl_delete)) + + # Remove ScanSpawnPoint entries associated to old spawnpoints. + num_rows = 0 + for i in range(0, num_records, step): + query = (ScanSpawnPoint + .delete() + .where((ScanSpawnPoint.spawnpoint << + old_sp[i:min(i + step, num_records)]))) + num_rows += query.execute() + log.debug('Deleted %d ScanSpawnPoint entries from old spawnpoints.', + num_rows) + + # Remove old and invalid SpawnPoint entries. + num_rows = 0 + for i in range(0, num_records, step): + query = (SpawnPoint + .delete() + .where((SpawnPoint.id << + old_sp[i:min(i + step, num_records)]))) + num_rows += query.execute() + log.debug('Deleted %d old SpawnPoint entries.', num_rows) + + sl_delete = list(sl_delete) + num_records = len(sl_delete) + + # Remove ScanSpawnPoint entries associated with old scanned locations. + num_rows = 0 + for i in range(0, num_records, step): + query = (ScanSpawnPoint + .delete() + .where((ScanSpawnPoint.scannedlocation << + sl_delete[i:min(i + step, num_records)]))) + num_rows += query.execute() + log.debug('Deleted %d ScanSpawnPoint entries from old scan locations.', + num_rows) + + # Remove ScannedLocation entries associated with old spawnpoints. + num_rows = 0 + for i in range(0, num_records, step): + query = (ScannedLocation + .delete() + .where((ScannedLocation.cellid << + sl_delete[i:min(i + step, num_records)]) & + (ScannedLocation.last_modified < + spawnpoint_timeout))) + num_rows += query.execute() + log.debug('Deleted %d ScannedLocation entries from old spawnpoints.', + num_rows) + + time_diff = default_timer() - start_timer + log.debug('Completed cleanup of old spawnpoint data in %.6f seconds.', + time_diff) + + +def db_clean_forts(age_hours): + log.debug('Beginning cleanup of old forts.') + start_timer = default_timer() + + fort_timeout = datetime.utcnow() - timedelta(hours=age_hours) + + with Gym.database().execution_context(): + # Remove old Gym entries. + query = (Gym + .delete() + .where(Gym.last_scanned < fort_timeout)) + rows = query.execute() + log.debug('Deleted %d old Gym entries.', rows) + + # Remove old Pokestop entries. + query = (Pokestop + .delete() + .where(Pokestop.last_updated < fort_timeout)) + rows = query.execute() + log.debug('Deleted %d old Pokestop entries.', rows) + + time_diff = default_timer() - start_timer + log.debug('Completed cleanup of old forts in %.6f seconds.', + time_diff) def bulk_upsert(cls, data, db): diff --git a/pogom/search.py b/pogom/search.py index a29090505b..f53e9e1610 100644 --- a/pogom/search.py +++ b/pogom/search.py @@ -1223,15 +1223,18 @@ def search_worker_thread(args, account_queue, account_sets, account_failures, def upsertKeys(keys, key_scheduler, db_updates_queue): - # Prepare hashing keys to be sent to the db. But only - # sent latest updates of the 'peak' value per key. + # Prepare hashing keys to be sent to the database. + # Keep highest peak value stored. hashkeys = {} - for key in keys: - key_instance = key_scheduler.keys[key] - hashkeys[key] = key_instance + stored_peaks = HashKeys.get_stored_peaks() + for key, instance in key_scheduler.keys.iteritems(): + hashkeys[key] = instance hashkeys[key]['key'] = key - hashkeys[key]['peak'] = max(key_instance['peak'], - HashKeys.getStoredPeak(key)) + + if key in stored_peaks: + max_peak = max(instance['peak'], stored_peaks[key]) + hashkeys[key]['peak'] = max_peak + db_updates_queue.put((HashKeys, hashkeys)) diff --git a/pogom/utils.py b/pogom/utils.py index 78f8426f74..414c20aee3 100644 --- a/pogom/utils.py +++ b/pogom/utils.py @@ -320,21 +320,17 @@ def get_args(): type=int, default=20) parser.add_argument('-kph', '--kph', help=('Set a maximum speed in km/hour for scanner ' + - 'movement. 0 to disable. Default: 35.'), + 'movement. Default: 35, 0 to disable.'), type=int, default=35) parser.add_argument('-hkph', '--hlvl-kph', help=('Set a maximum speed in km/hour for scanner ' + 'movement, for high-level (L30) accounts. ' + - '0 to disable. Default: 25.'), + 'Default: 25, 0 to disable.'), type=int, default=25) parser.add_argument('-ldur', '--lure-duration', help=('Change duration for lures set on pokestops. ' + 'This is useful for events that extend lure ' + 'duration.'), type=int, default=30) - parser.add_argument('-pd', '--purge-data', - help=('Clear Pokemon from database this many hours ' + - 'after they disappear (0 to disable).'), - type=int, default=0) parser.add_argument('-px', '--proxy', help='Proxy url (e.g. socks5://127.0.0.1:9050)', action='append') @@ -390,6 +386,35 @@ def get_args(): 'queue falls behind.'), type=int, default=1) + group = parser.add_argument_group('Database Cleanup') + group.add_argument('-DC', '--db-cleanup', + help='Enable regular database cleanup thread.', + action='store_true', default=False) + group.add_argument('-DCw', '--db-cleanup-worker', + help=('Clear worker status from database after X ' + + 'minutes of inactivity. ' + + 'Default: 30, 0 to disable.'), + type=int, default=30) + group.add_argument('-DCp', '--db-cleanup-pokemon', + help=('Clear pokemon from database X hours ' + + 'after they disappeared. ' + + 'Default: 0, 0 to disable.'), + type=int, default=0) + group.add_argument('-DCg', '--db-cleanup-gym', + help=('Clear gym details from database X hours ' + + 'after last gym scan. ' + + 'Default: 8, 0 to disable.'), + type=int, default=8) + group.add_argument('-DCs', '--db-cleanup-spawnpoint', + help=('Clear spawnpoint from database X hours ' + + 'after last valid scan. ' + + 'Default: 720, 0 to disable.'), + type=int, default=720) + group.add_argument('-DCf', '--db-cleanup-forts', + help=('Clear gyms and pokestops from database X days ' + + 'after last valid scan. ' + + 'Default: 0, 0 to disable.'), + type=int, default=0) parser.add_argument( '-wh', '--webhook', @@ -401,8 +426,6 @@ def get_args(): help=('Get all details about gyms (causes an ' + 'additional API hit for every gym).'), action='store_true', default=False) - parser.add_argument('-DC', '--enable-clean', help='Enable DB cleaner.', - action='store_true', default=False) parser.add_argument( '--wh-types', help=('Defines the type of messages to send to webhooks.'), @@ -459,8 +482,6 @@ def get_args(): parser.add_argument('-sn', '--status-name', default=str(os.getpid()), help=('Enable status page database update using ' + 'STATUS_NAME as main worker name.')) - parser.add_argument('-spp', '--status-page-password', default=None, - help='Set the status page password.') parser.add_argument('-hk', '--hash-key', default=None, action='append', help='Key for hash server') parser.add_argument('-novc', '--no-version-check', action='store_true', @@ -512,15 +533,22 @@ def get_args(): type=int, dest='verbose') rarity = parser.add_argument_group('Dynamic Rarity') rarity.add_argument('-Rh', '--rarity-hours', - help=('Number of hours of Pokemon data to use' + - ' to calculate dynamic rarity. Decimals' + - ' allowed. Default: 48. 0 to use all data.'), + help=('Number of hours of Pokemon data to use ' + + 'to calculate dynamic rarity. Decimals ' + + 'allowed. Default: 48, 0 to use all data.'), type=float, default=48) rarity.add_argument('-Rf', '--rarity-update-frequency', - help=('How often (in minutes) the dynamic rarity' + - ' should be updated. Decimals allowed.' + - ' Default: 0. 0 to disable.'), + help=('How often (in minutes) the dynamic rarity ' + + 'should be updated. Decimals allowed. ' + + 'Default: 0, 0 to disable.'), type=float, default=0) + statusp = parser.add_argument_group('Status Page') + statusp.add_argument('-SPp', '--status-page-password', default=None, + help='Set the status page password.') + statusp.add_argument('-SPf', '--status-page-filter', + help=('Filter worker status that are inactive for ' + + 'X minutes. Default: 30, 0 to disable.'), + type=int, default=30) parser.set_defaults(DEBUG=False) args = parser.parse_args() diff --git a/runserver.py b/runserver.py index ba9655ae88..b1a733941c 100755 --- a/runserver.py +++ b/runserver.py @@ -372,7 +372,7 @@ def main(): t.start() # Database cleaner; really only need one ever. - if args.enable_clean: + if args.db_cleanup: t = Thread(target=clean_db_loop, name='db-cleaner', args=(args,)) t.daemon = True t.start()