Minor bugfixes due to code refactor

- added docstrings to modules

lbsntransform/__main__.py

@@ -189,8 +189,9 @@ def set_logger():
     so we can also print to console while logging to file
     """
 
-    logging.basicConfig(handlers=[logging.FileHandler(
-        'log.log', 'w', 'utf-8')],
+    logging.basicConfig(
+        handlers=[logging.FileHandler(
+            'log.log', 'w', 'utf-8')],
         format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
         datefmt='%H:%M:%S',
         level=logging.DEBUG)

lbsntransform/classes/field_mapping_flickr.py

@@ -1,4 +1,9 @@
 # -*- coding: utf-8 -*-
+
+"""
+Module for mapping Flickr to common LBSN Structure.
+"""
+
 from .helper_functions import HelperFunctions as HF
 from .helper_functions import LBSNRecordDicts
 from lbsnstructure.lbsnstructure_pb2 import *

lbsntransform/classes/field_mapping_twitter.py

@@ -1,5 +1,9 @@
 # -*- coding: utf-8 -*-
 
+"""
+Module for mapping Twitter to common LBSN Structure.
+"""
+
 import shapely.geometry as geometry
 from shapely.geometry.polygon import Polygon
 import logging

lbsntransform/classes/helper_functions.py

@@ -1,5 +1,9 @@
 # -*- coding: utf-8 -*-
 
+"""
+Collection of helper functions being used in lbsntransform package.
+"""
+
 import re
 import csv
 import sys

lbsntransform/classes/load_data.py

@@ -1,4 +1,9 @@
 # -*- coding: utf-8 -*-
+
+"""
+Module for loding data from different sources (CSV, DB, JSON etc.).
+"""
+
 import sys
 import os
 import ntpath

lbsntransform/classes/shared_structure_proto_lbsndb.py

@@ -1,5 +1,9 @@
 # -*- coding: utf-8 -*-
 
+"""
+Shared structure and mapping between DB and Proto LBSN Structure.
+"""
+
 from .helper_functions import HelperFunctions as HF
 from lbsnstructure.lbsnstructure_pb2 import lbsnPost, \
     CompositeKey, \

lbsntransform/classes/store_csv.py

@@ -1,5 +1,9 @@
 # -*- coding: utf-8 -*-
 
+"""
+Module for storing common Proto LBSN Structure to CSV.
+"""
+
 import logging
 import traceback
 import os

lbsntransform/classes/submit_data.py

@@ -1,5 +1,9 @@
 # -*- coding: utf-8 -*-
 
+"""
+Module for storing common Proto LBSN Structure to PG DB.
+"""
+
 import logging
 from sys import exit
 import traceback

lbsntransform/config/config.py

@@ -1,4 +1,9 @@
 # -*- coding: utf-8 -*-
+
+"""
+Config module for parsing input args for lbsntransform package.
+"""
+
 import argparse
 import os
 import sys
@@ -26,7 +31,7 @@ def __init__(self):
         self.dbserveradress_output = None
         self.dbname_output = None
         self.transferlimit = None
-        self.transferCount = 50000
+        self.transfer_count = 50000
         self.number_of_records_to_fetch = 10000
         self.transfer_reactions = True
         self.disable_reactionpost_ref = False
@@ -112,13 +117,14 @@ def parseArgs(self):
                                    default=self.transferlimit,
                                    help='')
         settings_args.add_argument('-tC', "--transferCount",
-                                   default=self.transferCount,
-                                   help='Default to 50k: After how many parsed '
-                                   'records should the result be transferred to '
-                                   'the DB. Larger values improve speed, because '
-                                   'duplicate check happens in Python and not in '
-                                   'Postgres Coalesce; larger values are heavier '
-                                   'on memory.')
+                                   default=self.transfer_count,
+                                   help='Default to 50k: After how many '
+                                   'parsed records should the result be '
+                                   'transferred to the DB. Larger values '
+                                   'improve speed, because duplicate '
+                                   'check happens in Python and not in '
+                                   'Postgres Coalesce; larger values are '
+                                   'heavier on memory.')
         settings_args.add_argument('-nR', "--numberOfRecordsToFetch",
                                    default=self.number_of_records_to_fetch,
                                    help='')
@@ -145,7 +151,8 @@ def parseArgs(self):
         settings_args.add_argument('-rE', "--endWithDBRowNumber",
                                    default=self.endwith_db_rownumber,
                                    help='')
-        settings_args.add_argument('-d', "--debugMode", default=self.debug_mode,
+        settings_args.add_argument('-d', "--debugMode",
+                                   default=self.debug_mode,
                                    help='Needs to be implemented.')
         settings_args.add_argument('-gL', "--geocodeLocations",
                                    default=self.geocode_locations,
@@ -158,17 +165,20 @@ def parseArgs(self):
                                    help='Provide a list of input_source '
                                    'types that will be ignored (e.g. to '
                                    'ignore certain bots etc.)')
-        settings_args.add_argument('-iT', "--inputType", default=self.input_lbsn_type,
+        settings_args.add_argument('-iT', "--inputType",
+                                   default=self.input_lbsn_type,
                                    help='Input type, e.g. "post", "profile", '
                                    '"friendslist", "followerslist" etc.')
-        settings_args.add_argument('-mR', "--mapFullRelations", action='store_true',
+        settings_args.add_argument('-mR', "--mapFullRelations",
+                                   action='store_true',
                                    help='Defaults to False. Set to true '
                                    'to map full relations, e.g. many-to-many '
                                    'relationships such as user_follows, '
-                                   'user_friend, user_mentions etc. are mapped '
-                                   'in a separate table')
+                                   'user_friend, user_mentions etc. are '
+                                   'mapped in a separate table')
         settings_args.add_argument('-CSV', "--CSVOutput",
-                                   action='store_true', default=self.csv_output,
+                                   action='store_true',
+                                   default=self.csv_output,
                                    help='Set to True to Output all '
                                    'Submit values to CSV')
         settings_args.add_argument('-CSVal', "--CSVallowLinebreaks",
@@ -229,7 +239,7 @@ def parseArgs(self):
             if self.transferlimit == 0:
                 self.transferlimit = None
         if args.transferCount:
-            self.transferCount = int(args.transferCount)
+            self.transfer_count = int(args.transferCount)
         if args.numberOfRecordsToFetch:
             self.number_of_records_to_fetch = int(args.numberOfRecordsToFetch)
         if args.disableTransferReactions is True: