Skip to content

Commit

Permalink
refactor: clean up comments
Browse files Browse the repository at this point in the history
  • Loading branch information
@Sieboldianus
Sieboldianus committed Jun 3, 2019
1 parent f99397b commit cee0027
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 112 deletions.
96 changes: 0 additions & 96 deletions lbsntransform/__main__.py
View file
Expand Up @@ -125,102 +125,6 @@ def main():

input(f'Done. {how_long.stop_time()}')

# loop input DB until transferlimit reached or no more rows are returned
# while not finished:
# if config.transferlimit:
# max_records = config.transferlimit - processed_total
# else:
# max_records = None
# if config.is_local_input:
# if continue_number > len(loc_filelist) - 1:
# break
# records = LoadData.fetch_data_from_file(
# loc_filelist,
# continue_number,
# config.is_stacked_json,
# config.local_file_type,
# config.csv_delim)
# # skip empty files
# if not records:
# continue_number += 1
# continue
# else:
# records = \
# LoadData.fetch_json_data_from_lbsn(
# cursor_input,
# continue_number,
# max_records,
# config.number_of_records_to_fetch)
# if not records:
# break
# if config.is_local_input:
# continue_number += 1
# else:
# continue_number = records[-1][0] # last returned db_row_number
# processed_count, finished = \
# LoadData.loop_input_records(
# records,
# max_records,
# import_mapper,
# config)
# processed_records += processed_count
# processed_total += processed_count
# skipped_low_geoaccuracy_total += import_mapper.skipped_low_geoaccuracy
# print(f'{processed_total} input records processed (up to '
# f'{continue_number}). '
# f'Skipped {skipped_low_geoaccuracy} due to low geoaccuracy. '
# f'Count per type: {import_mapper.lbsn_records.get_type_counts()}'
# f'records.', end='\n')
#
# # update console
# # On the first loop or after 500.000 processed records,
# # transfer results to DB
# if not start_number or processed_records >= config.transfer_count or \
# finished:
# sys.stdout.flush()
# print(f'Storing {import_mapper.lbsn_records.count_glob} records.. '
# f'{HF.null_notice(import_mapper.null_island)})')
# output.store_lbsn_record_dicts(import_mapper)
# output.commit_changes()
# processed_records = 0
# # create a new empty dict of records
# import_mapper = importer(config.disable_reactionpost_ref,
# geocode_dict,
# config.map_relations,
# config.transfer_reactions,
# config.ignore_non_geotagged,
# ignore_sources_set,
# config.min_geoaccuracy)
# # remember the first processed DBRow ID
# if not start_number:
# if config.is_local_input:
# start_number = 1
# else:
# start_number = records[0][0] # first returned db_row_number

# submit remaining
# ??
# if import_mapper.lbsn_records.count_glob > 0:
# print(f'Transferring remaining '
# f'{import_mapper.lbsn_records.count_glob} to db.. '
# f'{HF.null_notice(import_mapper.null_island)})')
# output.store_lbsn_record_dicts(import_mapper)
# output.commit_changes()

# finalize all transactions (csv merge etc.)
# output.finalize()

# Close connections to DBs
# if not config.is_local_input:
# cursor_input.close()
# if config.dbuser_output:
# cursor_output.close()
# log.info(f'\n\nProcessed {processed_total} input records '
# f'(Input {start_number} to {continue_number}). '
# f'Skipped {skipped_low_geoaccuracy_total} '
# f'due to low geoaccuracy.')
# input(f'Done. {how_long.stop_time()}')


if __name__ == "__main__":
main()
26 changes: 10 additions & 16 deletions lbsntransform/classes/load_data.py
View file
Expand Up @@ -32,11 +32,14 @@ def __init__(
map_relations=None, transfer_reactions=None,
ignore_non_geotagged=None, min_geoaccuracy=None):
self.is_local_input = is_local_input
self.start_number = None
if not self.is_local_input:
# Start Value, Modify to continue from last processing
self.continue_number = startwith_db_rownumber
self.start_number = startwith_db_rownumber
else:
self.continue_number = skip_until_file
self.start_number = 1
if self.is_local_input:
self.filelist = LoadData._read_local_files(
input_path=input_path, recursive_load=recursive_load,
Expand Down Expand Up @@ -120,7 +123,7 @@ def _process_input(self, file_handle):
else:
while self.cursor:
record = self.fetch_json_data_from_lbsn(
self.cursor, self.start_id)
self.cursor, self.continue_number)
yield record

def convert_records(self, record_pipe):
Expand All @@ -129,9 +132,6 @@ def convert_records(self, record_pipe):
Returns statistic-counts, modifies (adds results to) import_mapper
"""
finished = False
processed_records = 0
db_row_number = 0
for record in record_pipe:
processed_records += 1
if self.is_local_input:
Expand All @@ -151,19 +151,8 @@ def convert_records(self, record_pipe):
single_record)
else:
exit(f'Format {self.local_file_type} not supportet.')

# if (self.transferlimit and self.processed_records >= self.transferlimit) or \
# (not self.is_local_input and
# self.endwith_db_rownumber and
# self.db_row_number >= self.endwith_db_rownumber):
# self.finished = True
# break
# return record as pipe
yield lbsn_records
# for records_dict in lbsn_records.all_dicts:
# type_name = records_dict[1]
# for lbsn_record in records_dict[0].values():
# yield lbsn_record, type_name
# return self.processed_records, finished

@staticmethod
def skip_empty_or_other(single_record):
Expand Down Expand Up @@ -201,6 +190,11 @@ def fetch_json_data_from_lbsn(self, cursor, start_id=0, get_max=None,
records = cursor.fetchall()
if cursor.rowcount == 0:
return None
# update last returned db_row_number
self.continue_number = records[-1][0]
if not self.start_number:
# first returned db_row_number
self.start_number = records[0][0]
for record in records:
return record

Expand Down

0 comments on commit cee0027

Please sign in to comment.