Cleaned up comments

classes/dbConnection.py

@@ -16,17 +16,17 @@ def __init__(self, serveradress=None, dbname=None,
         self.user = user
         self.password = password
         self.readonly = readonly
-        self.sslmode = sslmode #Choose 'disable' to connect without ssl
+        self.sslmode = sslmode # Choose 'disable' to connect without ssl
 
     def connect(self):
-        ##Database config
+        ## Database config
         conf = {
             "host":self.serveradress,
             "dbname":self.dbname,
             "user":self.user,
             "sslmode":self.sslmode
         }
-        ##Connect to database
+        ## Connect to database
         if self.password == 0:
             promt_txt = f"Enter the password for {conf['user']}: "
             if sys.stdin.isatty():
@@ -39,17 +39,14 @@ def connect(self):
         # define connection string
         conn_string = f"host='{conf['host']}' dbname='{conf['dbname']}' user='{conf['user']}' password='{conf['password']}' sslmode='{conf['sslmode']}'"
         # print connection string we will use to connect
-        #print("Connecting to database\n    ->%s" % (conn_string))
         # get a connection, if a connect cannot be made an exception will be raised here
         try:
             conn = psycopg2.connect(conn_string)
         except Exception as e:
             print(e)
             sys.exit
-        #conn.set_isolation_level(0)
         # conn.cursor will return a cursor object, you can use this cursor to perform queries
         cursor = conn.cursor()
         dnow = datetime.datetime.now()
         log.info(f'{dnow.strftime("%Y-%m-%d %H:%M:%S")} - Connected to {self.dbname}\n')
-        #print(f'{dnow.strftime("%Y-%m-%d %H:%M:%S")} - Connected to {self.dbname}\n')
         return conn,cursor

classes/fieldMapping.py

@@ -48,18 +48,17 @@ def parseJsonRecord(jsonStringDict,origin,lbsnRecords):
         if not userProfileImageURL == "http://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png":
             userRecord.profile_image_url = userProfileImageURL
 
-        #Some preprocessing for all types:
+        # Some preprocessing for all types:
         post_coordinates = jsonStringDict.get('coordinates') 
         if not post_coordinates:
-            #print("No Coordinates")
             l_lng = 0 
             l_lat = 0
         else:
             l_lng = post_coordinates.get('coordinates')[0]
             l_lat = post_coordinates.get('coordinates')[1]
             postGeoaccuracy = lbsnPost.LATLNG
 
-        #Check if Place is mentioned
+        # Check if Place is mentioned
         place = jsonStringDict.get('place')
         if place:
             placeID = place.get('id')
@@ -76,7 +75,6 @@ def parseJsonRecord(jsonStringDict,origin,lbsnRecords):
                 if not postGeoaccuracy:
                     postGeoaccuracy = lbsnPost.COUNTRY      
                 #log.debug(f'Placetype detected: country')
-                #sys.exit(place)
             elif place_type in ("city","neighborhood","admin"):
                 # city_guid
                 placeRecord = helperFunctions.createNewLBSNRecord_with_id(lbsnCity(),place.get('id'),origin)

classes/helperFunctions.py

@@ -121,7 +121,6 @@ def __init__(self, lbsnCountryDict=dict(), lbsnCityDict=dict(),
     def getTypeCounts(self):
         countList = []
         for x, y in self.KeyHashes.items():
-            #sys.exit(f'Length: {len(y)}')
             countList.append(f'{x}: {len(y)} ')
         return ''.join(countList)
 
@@ -175,7 +174,8 @@ def deepCompareMergeMessages(self,oldRecord,newRecord):
                             newEntries.remove(mainName)
                     x = getattr(oldRecord, descriptor.name)
                     x.extend(newEntries)
-                #elif descriptor.label == descriptor.TYPE_ENUM: 
+                # todo?
+                #elif descriptor.label == descriptor.TYPE_ENUM:
                 elif descriptor.type == descriptor.TYPE_MESSAGE:
                    x = getattr(oldRecord, descriptor.name)
                    x.CopyFrom(value_new)
@@ -208,6 +208,5 @@ def dictSelector(self, record):
         return dictSwitcher.get(record.DESCRIPTOR.name)
 
     def AddRecordToDict(self,record):
-        #print(type(record))
         dict = self.dictSelector(record)
         self.MergeExistingRecords(record,dict)

transferData.py

@@ -26,6 +26,8 @@ def import_config():
 
 def main():
     # Set Output to Replace in case of encoding issues (console/windows)
+    # Necessary? ProtoBuf will convert any problematic characters to Octal Escape Sequences anyway
+    # see https://stackoverflow.com/questions/23173340/how-to-convert-octal-escape-sequences-with-python
     sys.stdout = io.TextIOWrapper(sys.stdout.detach(), sys.stdout.encoding, 'replace')
     # Load Config
     # will be overwritten if args are given
@@ -96,22 +98,19 @@ def main():
     #for key,keyHash in lbsnRecords.KeyHashes.items():
     #    print(f'{key}: {", ".join(val for i, val in enumerate(random.sample(keyHash, min(10,len(keyHash)))))}')
     #print("Random Item for each type: \n")
-    print(f'lbsnCountry: {lbsnRecords.lbsnCountryDict[random.choice(list(lbsnRecords.lbsnCountryDict))]}')
-    print(f'lbsnCity: {lbsnRecords.lbsnCityDict[random.choice(list(lbsnRecords.lbsnCityDict))]}')
-    print(f'lbsnPlace: {lbsnRecords.lbsnPlaceDict[random.choice(list(lbsnRecords.lbsnPlaceDict))]}')
-    print(f'lbsnUser: {lbsnRecords.lbsnUserDict[random.choice(list(lbsnRecords.lbsnUserDict))]}')
-    print(f'lbsnPost: {lbsnRecords.lbsnPostDict[random.choice(list(lbsnRecords.lbsnPostDict))]}')
-    print(f'lbsnPostReaction: {lbsnRecords.lbsnPostReactionDict[random.choice(list(lbsnRecords.lbsnPostReactionDict))]}')  
+    #print(f'lbsnCountry: {lbsnRecords.lbsnCountryDict[random.choice(list(lbsnRecords.lbsnCountryDict))]}')
+    #print(f'lbsnCity: {lbsnRecords.lbsnCityDict[random.choice(list(lbsnRecords.lbsnCityDict))]}')
+    #print(f'lbsnPlace: {lbsnRecords.lbsnPlaceDict[random.choice(list(lbsnRecords.lbsnPlaceDict))]}')
+    #print(f'lbsnUser: {lbsnRecords.lbsnUserDict[random.choice(list(lbsnRecords.lbsnUserDict))]}')
+    #print(f'lbsnPost: {lbsnRecords.lbsnPostDict[random.choice(list(lbsnRecords.lbsnPostDict))]}')
+    #print(f'lbsnPostReaction: {lbsnRecords.lbsnPostReactionDict[random.choice(list(lbsnRecords.lbsnPostReactionDict))]}')  
     print('Done.')
 
 
 def loopInputRecords(jsonRecords, origin, processedRecords, transferlimit, finished, lbsnRecords):
-    #if not lbsnRecords:
-    #    lbsnRecords = lbsnRecordDicts()
     for record in jsonRecords:
         processedRecords += 1
         lastDBRowNumber = record[0]
-        #singleUJSONRecord = ujson.loads(record[2])
         singleJSONRecordDict = record[2]
         if singleJSONRecordDict.get('limit'):
             # Skip Rate Limiting Notice