Merge branch 'master' of https://github.com/oduwsdl/FollowerCountHistory

.gitignore

@@ -1,13 +1,11 @@
 follower/data/mementos.txt
 follower/data/NonParsedMementos.txt
-core/config/data/followercount
 */__pycache__
 */*/__pycache__
 *.pyc
 output/
 .RData
 dist/
-core/
 build/
 mementos.txt
 test.py

fch/core/config/configreader.py

@@ -0,0 +1,44 @@
+import pickle
+import os
+
+
+class ConfigurationReader:
+    """
+    This class is for reading Configuration of the program.
+
+    Attributes:
+        __conf_record (dict): Configuration Dictionary
+        mode (int): Sets mode of the program
+        start_time (int): Start Time of the analysis
+        end_time (int): End Time of the Analysis
+        out_dir (str): Default path for output
+        debug (bool): Sets Debug Mode
+        url_can (bool): Sets URL Canonicalization
+        tlangs (list): List of Twitter languages
+        dbname (str): Database Name
+        tdomain (list): List of Twitter Domains
+        tpath (list): List of Twitter paths
+        backup_logs (bool): Back Up Logs
+    """
+    def __init__(self, db_config=None):
+        """
+        This is the constructor for ConfigurationReader class.
+
+        Parameters:
+            db_config (str): Name of Configuration File
+        """
+        if not db_config:
+            self.db_config = "followercount"
+        with open(os.path.join(os.path.dirname(__file__), "data", self.db_config), "rb") as self.__ofile:
+            self.__conf_record = pickle.load(self.__ofile)
+            self.start_time = self.__conf_record["Start_Timestamp"]
+            self.end_time = self.__conf_record["End_Timestamp"]
+            self.out = self.__conf_record["Output_Dir"]
+            self.debug = self.__conf_record["Debug_Mode"]
+            self.tlangs = self.__conf_record["Twitter_Languages"]
+            self.dbname = self.__conf_record["Database"]
+            self.tdomain = self.__conf_record["Twitter_Domain"]
+            self.tpath = self.__conf_record["Twitter_Path"]
+            self.turl = self.__conf_record["Twitter_Url"]
+            self.frequency = self.__conf_record["Frequency"]
+            self.intermediate = self.__conf_record["Internediary_Dir"]

fch/core/config/configwriter.py

@@ -0,0 +1,39 @@
+import pickle
+import configparser
+import os
+
+
+class ConfigurationWriter:
+    """
+    This class is for writing Configuration of the program
+
+    Attributes:
+        __config (ConfigParser): ConfigParser object
+        __conf_path (str): Path to configuration file
+        __conf_record (dict): Stores configuration dictionary
+    """
+
+    def __init__(self, **kwargs):
+        """
+        This is constructor for ConfigurationWriter class
+
+        Parameters:
+            kwargs(list): Variable argument for configuration
+        """
+        self.__config = configparser.ConfigParser()
+        self.__conf_path = os.path.join(os.path.dirname(__file__), "data")
+        if len(self.__config.read(os.path.join(self.__conf_path, "config.ini"))) == 0:
+            self.__config.read(os.path.join(self.__conf_path, "config.ini"))
+        self.__conf_record = dict(Start_Timestamp=kwargs.get('st', int(self.__config['SETUP']['START_TIMESTAMP'])),
+                                  End_Timestamp=kwargs.get('et', int(self.__config['SETUP']['END_TIMESTAMP'])),
+                                  Output_Dir=kwargs.get('f', None),
+                                  Internediary_Dir=self.__config['SETUP']['INTERMIDIARY_DIR'],
+                                  Frequency=kwargs.get('freq', int(self.__config['SETUP']['FREQUENCY'])),
+                                  Database=kwargs.get("db_conf", "followercount"),
+                                  Debug_Mode=kwargs.get("debug", True if self.__config['SETUP']['DEBUG_MODE'] == "True" else False),
+                                  Twitter_Languages=self.__config['TWITTER']['LANGUAGES'].split(" "),
+                                  Twitter_Domain=self.__config['TWITTER']['DOMAIN'].split(" "),
+                                  Twitter_Path=self.__config['TWITTER']['PATH'],
+                                  Twitter_Url=self.__config['TWITTER']['URL'])
+        with open(os.path.join(self.__conf_path, self.__conf_record["Database"]), "wb") as ofile:
+            pickle.dump(self.__conf_record, ofile)

fch/core/config/data/README_config.txt

@@ -0,0 +1,70 @@
+********************
+[TWITTER]
+
+**********
+MODE:
+This is to set the mode of your analysis for finding deleted tweets.
+
+Supported Value:
+0: For getting deleted tweets based on Twitter api's usertimeline method. (Most recent 3200 tweets)
+1: For providing user defined start and end time for analysis
+**********
+
+**********
+START_TIMESTAMP:
+This sets the start timestamp for fetching mementos. It accepts value in Memento DateTime format (20190107235959)
+* Note: MODE 1 is required for using this option.
+**********
+
+**********
+END_TIMESTAMP:
+This sets the end timestamp for fetching mementos. It accepts value in Memento DateTime format (20190807235959)
+* Note: MODE 1 is required for using this option.
+**********
+
+
+**********
+URL_CANONICALIZATION: F
+This option allows for creating canonicalized Twitter URLs.
+For example, a Twitter URL will be appended with language variation and with_replies parameter.
+
+https://twitter.com/dougjones (1 URL)
+https://twitter.com/dougjones/with_replies (1 URL)
+https://twitter.com/dougjones?lang=en (47 URLs for 47 languages)
+https://twitter.com/dougjones/with_replies?lang=en (47 URLs for 47 languages)
+Total: 96 URLs for each URI-R
+
+Supported Value:
+T: Set to True
+F: Set to False
+**********
+
+
+********************
+[COMMON]
+********************
+
+**********
+OUTPUT_DIR:
+This options sets the default output directory.
+**********
+
+**********
+DEBUG_MODE:
+This option sets the dedug mode.
+
+Supported Value:
+True: Set to True
+False: Set to False
+**********
+
+********************
+[TWITTER_LANGUAGES]
+********************
+
+**********
+LANGUAGES: fr en ar ja es de it id pt ko tr ru nl fil ms zh-tw zh-cn hi no sv fi da pl hu fa he ur th uk ca ga el eu cs gl ro hr en-gb vi bn bg sr sk gu mr ta kn
+
+This option lists all the languages supported in Twitter URL. When a new language is encountered in the Twitter
+URL than already present in the current list add to the current list.
+**********

fch/core/config/data/config.ini

@@ -0,0 +1,13 @@
+[SETUP]
+START_TIMESTAMP: -1
+END_TIMESTAMP: -1
+OUTPUT: None
+INTERMIDIARY_DIR: /tmp
+DEBUG_MODE: False
+FREQUENCY: 0
+
+[TWITTER]
+LANGUAGES: fr en ar ja es de it id pt ko tr ru nl fil ms zh-tw zh-cn hi no sv fi da pl hu fa he ur th uk ca ga el eu cs gl ro hr en-gb vi bn bg sr sk gu mr ta kn
+DOMAIN: https://twitter.com https://mobile.twitter.com
+PATH: with_replies
+URL: scheme://domain/handle/path?lang=lang