Add restart time to FlockWatch output log. Add folder for collection …

…specific stopwords. Add x-axis label for time_analysis.py output

.gitignore

@@ -148,4 +148,6 @@ log/
 config.py
 data/
 time_analysis/figures/
-*.png
+*.png
+collection_stopwords/**
+!collection_stopwords/.keep

FlockWatch.py

@@ -416,10 +416,11 @@ def main():
         running = False
     elif cfg.FlockWatch_scheduling['repeat']:
         sleep_time = (cfg.FlockWatch_scheduling['repeat_interval'].hour * 60 * 60) + (cfg.FlockWatch_scheduling['repeat_interval'].minute * 60) - duration
+        resume_time = str((datetime.datetime.now() + datetime.timedelta(seconds=sleep_time)).time().replace(microsecond=0))
         if sleep_time < 0:
             logging.warning("FlockWatch takes too long to complete with your parameters for it to run as frequently as you want. FlockWatch will run again as soon as it can.\n")
         elif sleep_time > 0:
-            logging.info("FlockWatch is complete. Sleeping for {} seconds, then running again.\n".format(sleep_time))
+            logging.info("FlockWatch is complete. Sleeping for {0} seconds, resuming at {1}.\n".format(sleep_time, resume_time))
             time.sleep(sleep_time)
 
 """

time_analysis/time_analysis.py

@@ -31,8 +31,13 @@
 reports_in_range = []
 for f in log_dir_contents:
     log_date = parse(os.path.basename(f))
-    if log_date >= cfg.start_date:
-        reports_in_range.append(f)
+    if type(cfg.start_date) is datetime.datetime:
+        if log_date >= cfg.start_date:
+            reports_in_range.append(f)
+    elif type(cfg.start_date) is datetime.timedelta:
+        start_date = datetime.datetime.today().date() - cfg.start_date
+        if log_date >= start_date:
+            reports_in_range.append(f)
 logging.info("Found {} sets of results.".format(len(reports_in_range)))
 
 co_occurrence_reports = []
@@ -63,13 +68,22 @@
     plot_df = co_occurrence_over_time.loc[term]
     plot_df = plot_df.transpose().sort_index()
     plot_df.sort_values(by=list(plot_df.columns), ascending=False, inplace=True)
+    plot_dates = []
+    plot_dates = set(plot_dates)
     plot_df = plot_df.iloc[:,:5]
+    plot_df.sort_index(inplace=True)
     co_occurrence_plot = plt.figure(figsize=(10, 4))
-    plot_df.plot(ylim=(0, 1))
+    plot1 = co_occurrence_plot.add_subplot(111)
+    for cos in plot_df.columns.values:
+        plot1.plot_date(plot_df.index, plot_df[cos], fmt='-')
+    #plot1 = plot_df.plot(ylim=(0, 1))
+    plot1.set_ylim(0,1)
+    co_occurrence_plot.autofmt_xdate()
+    co_occurrence_plot.legend()
     fig_name = term+'-co-occurrence-plot'
     fig_name = os.path.join(figure_folder, fig_name)
     plt.savefig(fig_name)
     #plt.show()
-    plt.close()
+    plt.close('all')
 
 logging.info("Plots of top co-occurring terms created at {}.\nProcess complete.".format(figure_folder))

time_analysis/time_config.py

@@ -1,6 +1,6 @@
 import datetime
 
-start_date = datetime.datetime(2018, 6, 1)
+start_date = datetime.timedelta(days=7)  # datetime.datetime(2018, 6, 1)
 
 time_analysis_log_file = './time.log'
 log_folder = '../log/'