From 52608e421fc563ab3b59b5ffdc31123d529ad269 Mon Sep 17 00:00:00 2001 From: AD Date: Thu, 3 Jan 2019 17:09:16 +0100 Subject: [PATCH] Minor bugfixes due to code refactor - added docstrings to modules --- lbsntransform/__main__.py | 5 ++- lbsntransform/classes/field_mapping_flickr.py | 5 +++ .../classes/field_mapping_twitter.py | 4 ++ lbsntransform/classes/helper_functions.py | 4 ++ lbsntransform/classes/load_data.py | 5 +++ .../classes/shared_structure_proto_lbsndb.py | 4 ++ lbsntransform/classes/store_csv.py | 4 ++ lbsntransform/classes/submit_data.py | 4 ++ lbsntransform/config/config.py | 40 ++++++++++++------- 9 files changed, 58 insertions(+), 17 deletions(-) diff --git a/lbsntransform/__main__.py b/lbsntransform/__main__.py index b9e9356..1124854 100644 --- a/lbsntransform/__main__.py +++ b/lbsntransform/__main__.py @@ -189,8 +189,9 @@ def set_logger(): so we can also print to console while logging to file """ - logging.basicConfig(handlers=[logging.FileHandler( - 'log.log', 'w', 'utf-8')], + logging.basicConfig( + handlers=[logging.FileHandler( + 'log.log', 'w', 'utf-8')], format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s', datefmt='%H:%M:%S', level=logging.DEBUG) diff --git a/lbsntransform/classes/field_mapping_flickr.py b/lbsntransform/classes/field_mapping_flickr.py index fd08995..32d4a35 100644 --- a/lbsntransform/classes/field_mapping_flickr.py +++ b/lbsntransform/classes/field_mapping_flickr.py @@ -1,4 +1,9 @@ # -*- coding: utf-8 -*- + +""" +Module for mapping Flickr to common LBSN Structure. +""" + from .helper_functions import HelperFunctions as HF from .helper_functions import LBSNRecordDicts from lbsnstructure.lbsnstructure_pb2 import * diff --git a/lbsntransform/classes/field_mapping_twitter.py b/lbsntransform/classes/field_mapping_twitter.py index 0976a8a..ea50cae 100644 --- a/lbsntransform/classes/field_mapping_twitter.py +++ b/lbsntransform/classes/field_mapping_twitter.py @@ -1,5 +1,9 @@ # -*- coding: utf-8 -*- +""" +Module for mapping Twitter to common LBSN Structure. +""" + import shapely.geometry as geometry from shapely.geometry.polygon import Polygon import logging diff --git a/lbsntransform/classes/helper_functions.py b/lbsntransform/classes/helper_functions.py index 3e21113..7d7a718 100644 --- a/lbsntransform/classes/helper_functions.py +++ b/lbsntransform/classes/helper_functions.py @@ -1,5 +1,9 @@ # -*- coding: utf-8 -*- +""" +Collection of helper functions being used in lbsntransform package. +""" + import re import csv import sys diff --git a/lbsntransform/classes/load_data.py b/lbsntransform/classes/load_data.py index a6cb920..63878c4 100644 --- a/lbsntransform/classes/load_data.py +++ b/lbsntransform/classes/load_data.py @@ -1,4 +1,9 @@ # -*- coding: utf-8 -*- + +""" +Module for loding data from different sources (CSV, DB, JSON etc.). +""" + import sys import os import ntpath diff --git a/lbsntransform/classes/shared_structure_proto_lbsndb.py b/lbsntransform/classes/shared_structure_proto_lbsndb.py index 83688a5..e00d903 100644 --- a/lbsntransform/classes/shared_structure_proto_lbsndb.py +++ b/lbsntransform/classes/shared_structure_proto_lbsndb.py @@ -1,5 +1,9 @@ # -*- coding: utf-8 -*- +""" +Shared structure and mapping between DB and Proto LBSN Structure. +""" + from .helper_functions import HelperFunctions as HF from lbsnstructure.lbsnstructure_pb2 import lbsnPost, \ CompositeKey, \ diff --git a/lbsntransform/classes/store_csv.py b/lbsntransform/classes/store_csv.py index 1df4aab..065b477 100644 --- a/lbsntransform/classes/store_csv.py +++ b/lbsntransform/classes/store_csv.py @@ -1,5 +1,9 @@ # -*- coding: utf-8 -*- +""" +Module for storing common Proto LBSN Structure to CSV. +""" + import logging import traceback import os diff --git a/lbsntransform/classes/submit_data.py b/lbsntransform/classes/submit_data.py index ab8145f..6c79956 100644 --- a/lbsntransform/classes/submit_data.py +++ b/lbsntransform/classes/submit_data.py @@ -1,5 +1,9 @@ # -*- coding: utf-8 -*- +""" +Module for storing common Proto LBSN Structure to PG DB. +""" + import logging from sys import exit import traceback diff --git a/lbsntransform/config/config.py b/lbsntransform/config/config.py index 358f121..24ad867 100644 --- a/lbsntransform/config/config.py +++ b/lbsntransform/config/config.py @@ -1,4 +1,9 @@ # -*- coding: utf-8 -*- + +""" +Config module for parsing input args for lbsntransform package. +""" + import argparse import os import sys @@ -26,7 +31,7 @@ def __init__(self): self.dbserveradress_output = None self.dbname_output = None self.transferlimit = None - self.transferCount = 50000 + self.transfer_count = 50000 self.number_of_records_to_fetch = 10000 self.transfer_reactions = True self.disable_reactionpost_ref = False @@ -112,13 +117,14 @@ def parseArgs(self): default=self.transferlimit, help='') settings_args.add_argument('-tC', "--transferCount", - default=self.transferCount, - help='Default to 50k: After how many parsed ' - 'records should the result be transferred to ' - 'the DB. Larger values improve speed, because ' - 'duplicate check happens in Python and not in ' - 'Postgres Coalesce; larger values are heavier ' - 'on memory.') + default=self.transfer_count, + help='Default to 50k: After how many ' + 'parsed records should the result be ' + 'transferred to the DB. Larger values ' + 'improve speed, because duplicate ' + 'check happens in Python and not in ' + 'Postgres Coalesce; larger values are ' + 'heavier on memory.') settings_args.add_argument('-nR', "--numberOfRecordsToFetch", default=self.number_of_records_to_fetch, help='') @@ -145,7 +151,8 @@ def parseArgs(self): settings_args.add_argument('-rE', "--endWithDBRowNumber", default=self.endwith_db_rownumber, help='') - settings_args.add_argument('-d', "--debugMode", default=self.debug_mode, + settings_args.add_argument('-d', "--debugMode", + default=self.debug_mode, help='Needs to be implemented.') settings_args.add_argument('-gL', "--geocodeLocations", default=self.geocode_locations, @@ -158,17 +165,20 @@ def parseArgs(self): help='Provide a list of input_source ' 'types that will be ignored (e.g. to ' 'ignore certain bots etc.)') - settings_args.add_argument('-iT', "--inputType", default=self.input_lbsn_type, + settings_args.add_argument('-iT', "--inputType", + default=self.input_lbsn_type, help='Input type, e.g. "post", "profile", ' '"friendslist", "followerslist" etc.') - settings_args.add_argument('-mR', "--mapFullRelations", action='store_true', + settings_args.add_argument('-mR', "--mapFullRelations", + action='store_true', help='Defaults to False. Set to true ' 'to map full relations, e.g. many-to-many ' 'relationships such as user_follows, ' - 'user_friend, user_mentions etc. are mapped ' - 'in a separate table') + 'user_friend, user_mentions etc. are ' + 'mapped in a separate table') settings_args.add_argument('-CSV', "--CSVOutput", - action='store_true', default=self.csv_output, + action='store_true', + default=self.csv_output, help='Set to True to Output all ' 'Submit values to CSV') settings_args.add_argument('-CSVal', "--CSVallowLinebreaks", @@ -229,7 +239,7 @@ def parseArgs(self): if self.transferlimit == 0: self.transferlimit = None if args.transferCount: - self.transferCount = int(args.transferCount) + self.transfer_count = int(args.transferCount) if args.numberOfRecordsToFetch: self.number_of_records_to_fetch = int(args.numberOfRecordsToFetch) if args.disableTransferReactions is True: