refactor: untangle classes in helper_functions

lbsntransform/__init__.py

@@ -4,10 +4,11 @@
 
 from .version import __version__
 
-from .lbsntransform_ import LBSNTransform
-from .tools.helper_functions import (GeocodeLocations, HelperFunctions,
-                                     LBSNRecordDicts, TimeMonitor)
+from .config.config import BaseConfig
 from .input.load_data import LoadData
-from .output.lbsn.shared_structure_proto_lbsndb import ProtoLBSNMapping
+from .lbsntransform_ import LBSNTransform
 from .output.hll.shared_structure_proto_hlldb import ProtoHLLMapping
-from .config.config import BaseConfig
+from .output.lbsn.shared_structure_proto_lbsndb import ProtoLBSNMapping
+from .output.shared_structure import (GeocodeLocations, LBSNRecordDicts,
+                                      TimeMonitor)
+from .tools.helper_functions import HelperFunctions as HF

lbsntransform/__main__.py

@@ -22,7 +22,7 @@
 import traceback
 
 from lbsntransform.tools.helper_functions import HelperFunctions as HF
-from lbsntransform.tools.helper_functions import TimeMonitor
+from lbsntransform.output.shared_structure import TimeMonitor
 from lbsntransform.input.load_data import LoadData
 from lbsntransform.output.submit_data import LBSNTransfer
 from lbsntransform.config.config import BaseConfig

lbsntransform/input/load_data.py

@@ -22,7 +22,7 @@
 from lbsnstructure import lbsnstructure_pb2 as lbsn
 
 from ..tools.db_connection import DBConnection
-from ..tools.helper_functions import GeocodeLocations
+from ..output.shared_structure import GeocodeLocations
 from ..tools.helper_functions import HelperFunctions as HF
 
 

lbsntransform/input/mappings/field_mapping_fb.py

@@ -18,7 +18,7 @@
 from shapely.geometry.polygon import Polygon
 
 from ...tools.helper_functions import HelperFunctions as HF
-from ...tools.helper_functions import LBSNRecordDicts
+from ...output.shared_structure import LBSNRecordDicts
 
 
 class FieldMappingFBPlace():

lbsntransform/input/mappings/field_mapping_flickr.py

@@ -14,7 +14,7 @@
 from lbsnstructure import lbsnstructure_pb2 as lbsn
 
 from ...tools.helper_functions import HelperFunctions as HF
-from ...tools.helper_functions import LBSNRecordDicts
+from ...output.shared_structure import LBSNRecordDicts
 
 
 class FieldMappingFlickr():

lbsntransform/input/mappings/field_mapping_twitter.py

@@ -19,7 +19,7 @@
 from shapely.geometry.polygon import Polygon
 
 from ...tools.helper_functions import HelperFunctions as HF
-from ...tools.helper_functions import LBSNRecordDicts
+from ...output.shared_structure import LBSNRecordDicts
 
 
 class FieldMappingTwitter():

lbsntransform/input/mappings/field_mapping_yfcc100m.py

@@ -18,7 +18,7 @@
 from lbsnstructure import lbsnstructure_pb2 as lbsn
 
 from ...tools.helper_functions import HelperFunctions as HF
-from ...tools.helper_functions import LBSNRecordDicts
+from ...output.shared_structure import LBSNRecordDicts
 
 # pylint: disable=no-member
 

lbsntransform/lbsntransform_.py

@@ -16,7 +16,7 @@
 from pathlib import Path
 
 from .tools.helper_functions import HelperFunctions as HF
-from .tools.helper_functions import LBSNRecordDicts
+from .output.shared_structure import LBSNRecordDicts
 from .input.load_data import LoadData
 from .output.submit_data import LBSNTransfer
 from .config.config import BaseConfig

lbsntransform/output/csv/store_csv.py

@@ -21,8 +21,10 @@
 from google.protobuf import text_format
 from google.protobuf.timestamp_pb2 import Timestamp
 
-from lbsntransform.tools.helper_functions import HelperFunctions, LBSNRecordDicts
-from lbsntransform.output.lbsn.shared_structure_proto_lbsndb import ProtoLBSNMapping
+from lbsntransform.output.lbsn.shared_structure_proto_lbsndb import \
+    ProtoLBSNMapping
+from lbsntransform.tools.helper_functions import HelperFunctions as HF
+from lbsntransform.output.shared_structure import LBSNRecordDicts
 
 
 class LBSNcsv():
@@ -160,7 +162,7 @@ def merge_files(self, filelist, type_name):
     def create_proto_by_descriptor_name(self, desc_name):
         """Create new proto record by name
         """
-        new_record = HelperFunctions.dict_type_switcher(desc_name)
+        new_record = HF.dict_type_switcher(desc_name)
         return new_record
 
     # def parse_message(self,msgType,stringMessage):
@@ -227,7 +229,7 @@ def merge_records(self, duplicate_record_lines, type_name):
                     record = self.get_record_from_base64_encoded_string(
                         duprecord, type_name)
                     # will modify/overwrite prev_duprecord
-                    HelperFunctions.merge_existing_records(
+                    HF.merge_existing_records(
                         prev_duprecord, record)
                 merged_record = self.serialize_encode_record(prev_duprecord)
             else:

lbsntransform/output/hll/base/social.py

@@ -3,13 +3,14 @@
 """Hll bases for social facet
 """
 
-from collections import namedtuple, OrderedDict
-from typing import Tuple, List, Any, Dict, Union
+from collections import OrderedDict, namedtuple
+from typing import Any, Dict, List, Tuple, Union
 
-from lbsntransform.tools.helper_functions import HelperFunctions as HF
-from ..hll_functions import HLLFunctions as HLF
 from lbsnstructure import lbsnstructure_pb2 as lbsn
 from lbsntransform.output.hll import hll_bases as hll
+from lbsntransform.tools.helper_functions import HelperFunctions as HF
+
+from ..hll_functions import HLLFunctions as HLF
 
 FACET = 'social'
 

lbsntransform/output/hll/base/spatial.py

@@ -3,13 +3,14 @@
 """Hll bases for spatial facet
 """
 
-from collections import namedtuple, OrderedDict
-from typing import Tuple, List, Any, Dict, Union
+from collections import OrderedDict, namedtuple
+from typing import Any, Dict, List, Tuple, Union
 
-from lbsntransform.tools.helper_functions import HelperFunctions as HF
-from ..hll_functions import HLLFunctions as HLF
 from lbsnstructure import lbsnstructure_pb2 as lbsn
 from lbsntransform.output.hll import hll_bases as hll
+from lbsntransform.tools.helper_functions import HelperFunctions as HF
+
+from ..hll_functions import HLLFunctions as HLF
 
 FACET = 'spatial'
 

lbsntransform/output/hll/base/temporal.py

@@ -3,13 +3,14 @@
 """Hll bases for temporal facet
 """
 
-from collections import namedtuple, OrderedDict
-from typing import Tuple, List, Any, Dict, Union
+from collections import OrderedDict, namedtuple
+from typing import Any, Dict, List, Tuple, Union
 
-from lbsntransform.tools.helper_functions import HelperFunctions as HF
-from ..hll_functions import HLLFunctions as HLF
 from lbsnstructure import lbsnstructure_pb2 as lbsn
 from lbsntransform.output.hll import hll_bases as hll
+from lbsntransform.tools.helper_functions import HelperFunctions as HF
+
+from ..hll_functions import HLLFunctions as HLF
 
 FACET = 'temporal'
 

lbsntransform/output/hll/base/topical.py

@@ -3,13 +3,14 @@
 """Hll bases for topical facet
 """
 
-from collections import namedtuple, OrderedDict
-from typing import Tuple, List, Any, Dict, Union
+from collections import OrderedDict, namedtuple
+from typing import Any, Dict, List, Tuple, Union
 
-from lbsntransform.tools.helper_functions import HelperFunctions as HF
-from ..hll_functions import HLLFunctions as HLF
 from lbsnstructure import lbsnstructure_pb2 as lbsn
 from lbsntransform.output.hll import hll_bases as hll
+from lbsntransform.tools.helper_functions import HelperFunctions as HF
+
+from ..hll_functions import HLLFunctions as HLF
 
 FACET = 'topical'
 

lbsntransform/output/hll/hll_bases.py

@@ -54,14 +54,14 @@ class structures defined here, thus it is important that order of keys,
 A privacy-aware model to process data from location-based social media.
 """
 
-import sys
 import inspect
-from collections import namedtuple, OrderedDict
-from typing import Tuple, List, Any, Dict, Union
+import sys
+from collections import OrderedDict, namedtuple
+from typing import Any, Dict, List, Tuple, Union
 
 from lbsnstructure import lbsnstructure_pb2 as lbsn
-
 from lbsntransform.tools.helper_functions import HelperFunctions as HF
+
 from .hll_functions import HLLFunctions as HLF
 
 # named tuple of defined hll metrics

lbsntransform/output/hll/hll_functions.py

@@ -6,7 +6,8 @@
 
 import datetime as dt
 from collections import namedtuple
-from typing import Any, Generator, List, Set, Tuple, Union, Dict
+from typing import Any, Dict, Generator, List, Set, Tuple, Union
+
 from lbsnstructure import lbsnstructure_pb2 as lbsn
 from lbsntransform.tools.helper_functions import HelperFunctions as HF
 

lbsntransform/output/hll/shared_structure_proto_hlldb.py

@@ -8,14 +8,15 @@
 
 import inspect
 import sys
-from typing import Tuple, List, Any, Dict, Union
 from collections import defaultdict, namedtuple
+from typing import Any, Dict, List, Tuple, Union
 
 from lbsnstructure import lbsnstructure_pb2 as lbsn
+from lbsntransform.output.hll import hll_bases as hll
+from lbsntransform.output.hll.base import social, spatial, temporal, topical
 from lbsntransform.tools.helper_functions import HelperFunctions as HF
+
 from .hll_functions import HLLFunctions as HLF
-from lbsntransform.output.hll.base import social, spatial, temporal, topical
-from lbsntransform.output.hll import hll_bases as hll
 
 
 class ProtoHLLMapping():

lbsntransform/output/hll/sql_hll.py

@@ -4,8 +4,8 @@
 Module for sql insert functions for LBSN (hll) db
 """
 
-from lbsntransform.tools.helper_functions import HelperFunctions as HF
 from lbsntransform.output.hll import hll_bases as hll
+from lbsntransform.tools.helper_functions import HelperFunctions as HF
 
 
 class HLLSql():