Merge pull request #91 from lsst-dm/tickets/DM-36564

Tickets/dm 36564

milestones.py

@@ -5,6 +5,8 @@
 from datetime import datetime
 
 import milestones
+from milestones.excel import load_f2due_pmcs_excel
+from milestones.utility import get_pmcs_path_months
 
 
 def parse_args():
@@ -22,7 +24,6 @@ def parse_args():
         default=milestones.get_local_data_path(),
     )
     parser.add_argument("--verbose", "-v", action="count", default=0)
-    parser.add_argument("--forecast", "-f", action="store_true", help="Use end_date")
 
     subparsers = parser.add_subparsers(title="Output targets")
 
@@ -67,6 +68,10 @@ def parse_args():
         "--prefix", help="List of prefixes for burndown milestones.",
         default="DM- DLP- LDM-503-"
     )
+    burndown.add_argument(
+        "--months", help="Specify number of months prior to use as forecast",
+        type=int, default=0
+    )
 
     csv = subparsers.add_parser(
         "csv", help="Generate a CSV version of the milestone schedule."
@@ -138,9 +143,9 @@ def parse_args():
 if __name__ == "__main__":
     args = parse_args()
     print("Working with "+args.pmcs_data)
-    milestones = milestones.load_milestones(args.pmcs_data, args.local_data,
-                                            args.forecast)
-    if args.forecast:
-        args.output = f"fcast_{args.output}"
+    milestones = milestones.load_milestones(args.pmcs_data, args.local_data)
+    if "months" in args and args.months > 0:
+        fpath = get_pmcs_path_months(args.pmcs_data, args.months)
+        load_f2due_pmcs_excel(fpath, milestones)
 
     args.func(args, milestones)

milestones/burndown.py

@@ -13,7 +13,8 @@ def burndown(args, milestones):
 
     prefixes = args.prefix.split()
 
-    print(f"Burndown for milestones starting with {prefixes}")
+    print(f"Burndown for milestones starting with {prefixes} using "
+          f"{args.months} prior forcast")
 
     milestones = [
         ms
@@ -24,6 +25,8 @@ def burndown(args, milestones):
         and (not ms.completed or ms.completed > start_date)
     ]
 
+    dofcastline = args.months > 0
+
     month_starts = []
     for year in range(start_date.year, end_date.year + 1):
         for month in range(1, 13):
@@ -33,15 +36,23 @@ def burndown(args, milestones):
 
     model = []
     actual = []
+    fcast = []
+    f2cast = []
     for date in month_starts:
-        model_remain = actual_remain = len(milestones)
+        f2cast_remain = fcast_remain = model_remain = actual_remain = len(milestones)
         for ms in milestones:
+            if dofcastline and ms.f2due <= date:
+                f2cast_remain -= 1
+            if ms.fdue <= date:
+                fcast_remain -= 1
             if ms.due <= date:
                 model_remain -= 1
             if ms.completed and ms.completed <= date:
                 actual_remain -= 1
 
         model.append(model_remain)
+        f2cast.append(f2cast_remain)
+        fcast.append(fcast_remain)
         actual.append(actual_remain)
 
     last_achieved_month = None
@@ -52,6 +63,9 @@ def burndown(args, milestones):
             last_achieved_month = ms.completed
 
     plt.plot(month_starts, model, label="Baseline")
+    if dofcastline:
+        plt.plot(month_starts, f2cast, label=f"-{args.months}m Forecast")
+        plt.plot(month_starts, fcast, label="Forecast")
 
     achieved_months = [mnth for mnth in month_starts if mnth <= last_achieved_month]
     # Need to acount for year wrap

milestones/excel.py

@@ -44,7 +44,32 @@ def extract_wbs(value):
         return ""
 
 
-def extract_task_details(task_sheet, forecast=False):
+def extract_fcast(task_sheet, milestones):
+    # get forceast dates from sheet
+    # go through all milestones and add the f2date
+    assert task_sheet.name == TASK_SHEET_NAME
+    fetcher = CellFetcher(task_sheet.row(0))
+    fdates = {}
+    for rownum in range(START_ROW, task_sheet.nrows):
+        d = fetcher("end_date", task_sheet.row(rownum))
+        code = fetcher("task_code", task_sheet.row(rownum))
+        try:
+            f2due = extract_date(d)
+            fdates[code] = f2due
+        except ValueError:
+            pass
+
+    print(f"Got {len(fdates)} forecast dates")
+    for m in milestones:
+        if m.code in fdates:
+            m.f2due = fdates[m.code]
+        else:  # could be a new milestone not there n months ago
+            m.f2due = m.due
+
+    return milestones
+
+
+def extract_task_details(task_sheet):
     assert task_sheet.name == TASK_SHEET_NAME
     milestones = list()
     fetcher = CellFetcher(task_sheet.row(0))
@@ -65,11 +90,9 @@ def extract_task_details(task_sheet, forecast=False):
         #                   will be the same as the end_date for zero duration
         #                   activities like milestones.
         #
-        # We use the first available. Unless we want forecast
+        # We use the first available.
 
         date_order = ["base_end_date", "end_date", "start_date"]
-        if forecast:
-            date_order = ["end_date", "base_end_date", "start_date"]
 
         for date_field in (date_order):
             d = fetcher(date_field, task_sheet.row(rownum))
@@ -85,10 +108,9 @@ def extract_task_details(task_sheet, forecast=False):
         base_end_date = fetcher("base_end_date", task_sheet.row(rownum))
         start_date = fetcher("start_date", task_sheet.row(rownum))
         end_date = fetcher("end_date", task_sheet.row(rownum))
-        try:
+
+        if (end_date):
             fdue = extract_date(end_date)
-        except ValueError:
-            fdue = due
 
         completed = None
         if status == "Completed":
@@ -124,8 +146,17 @@ def set_successors(milestones, relation_sheet):
             ms.predecessors.add(preds[i])
 
 
-def load_pmcs_excel(path, forecast):
+def load_pmcs_excel(path):
     workbook = xlrd.open_workbook(path, logfile=sys.stderr)
-    milestones = extract_task_details(workbook.sheets()[0], forecast)
+    milestones = extract_task_details(workbook.sheets()[0])
     set_successors(milestones, workbook.sheets()[1])
     return milestones
+
+
+def load_f2due_pmcs_excel(fpath, milestones):
+    # given milestones, load the sheet from N months prior
+    # set fdue2 to the forecast date from the file
+    print(f"Loading forecast from {fpath}")
+    workbook = xlrd.open_workbook(fpath, logfile=sys.stderr)
+    milestones = extract_fcast(workbook.sheets()[0], milestones)
+    return milestones

milestones/milestone.py

@@ -32,6 +32,8 @@ class Milestone(object):
     jira: Optional[str] = None
     jira_testplan: Optional[str] = None
 
+    f2due: Optional[datetime] = None
+
     @property
     def short_name(self):
         return self._short_name if self._short_name else self.name

milestones/utility.py

@@ -15,6 +15,7 @@
     "add_rst_citations",
     "escape_latex",
     "format_latex",
+    "get_pmcs_path_months",
     "get_latest_pmcs_path",
     "get_local_data_path",
     "load_milestones",
@@ -50,6 +51,17 @@
 # baseline) or "ME" (for forecast).
 
 
+def get_pmcs_path_months(cpath=None, months=3):
+    """Get the list of pmcs files - find the one passed and take the one months prior.
+    """
+    path = os.path.normpath(os.path.join(os.path.dirname(__file__), "..",
+                                         "data", "pmcs"))
+    all_files = sorted(glob.glob(os.path.join(path, "??????-ME.xls")))
+    for ind, f in enumerate(all_files):
+        if f.__contains__(cpath) and ind >= months:
+            return all_files[ind - months]
+
+
 def get_latest_pmcs_path(path=None):
     """By default, fetch the latest forecast.
     """
@@ -107,12 +119,12 @@ def add_rst_citations(text, cite_handles=DOC_HANDLES):
     return add_citations(text, cite_handles, r"\1 :cite:`\1`")
 
 
-def load_milestones(pmcs_filename, local_data_filename, forecast=False):
+def load_milestones(pmcs_filename, local_data_filename):
     logger = logging.getLogger(__name__)
 
     logger.info(f"Loading PMCS data from: {pmcs_filename}")
     logger.info(f"Loading local annotations from: {local_data_filename}")
-    milestones = load_pmcs_excel(pmcs_filename, forecast)
+    milestones = load_pmcs_excel(pmcs_filename)
 
     with open(local_data_filename) as f:
         local = yaml.safe_load(f)