leaderboard exps for lptm

config/lptm.json

@@ -0,0 +1,11 @@
+{
+    "config": {
+        "task_name": "forecasting2",
+        "forecast_horizon": 192,
+        "head_dropout": 0.1,
+        "weight_decay": 0,
+        "freeze_encoder": true, 
+        "freeze_embedder": true,
+        "freeze_head": false
+    }
+}

config/timemoe.json

@@ -0,0 +1,25 @@
+{
+    "repo": "Maple728/TimeMoE-50M",
+    "config": {
+        "input_size": 1,
+        "hidden_size":  4096,
+        "intermediate_size":  22016,
+        "horizon_lengths": 1,
+        "num_hidden_layers": 32,
+        "num_attention_heads": 32,
+        "num_key_value_heads": null,
+        "hidden_act": "silu",
+        "num_experts_per_tok": 2,
+        "num_experts": 1,
+        "max_position_embeddings": 32768,
+        "initializer_range": 0.02,
+        "rms_norm_eps": 1e-6,
+        "use_cache": true,
+        "use_dense": false,
+        "rope_theta": 10000,
+        "attention_dropout": 0.0,
+        "apply_aux_loss": true,
+        "router_aux_loss_factor": 0.02,
+        "tie_word_embeddings": false
+    }
+}

leaderboard.py

@@ -3,14 +3,16 @@
 import numpy as np
 import pandas as pd
 import time
+import torch
+import gc
 
 src_path = os.path.abspath(os.path.join("src"))
 if src_path not in sys.path:
     sys.path.insert(0, src_path)
 
-from samay.model import TimesfmModel, MomentModel, ChronosModel, ChronosBoltModel, TinyTimeMixerModel, MoiraiTSModel
-from samay.dataset import TimesfmDataset, MomentDataset, ChronosDataset, ChronosBoltDataset, TinyTimeMixerDataset, MoiraiDataset
-from samay.utils import load_args, get_gifteval_datasets
+from samay.model import TimesfmModel, MomentModel, ChronosModel, ChronosBoltModel, TinyTimeMixerModel, MoiraiTSModel, LPTMModel
+from samay.dataset import TimesfmDataset, MomentDataset, ChronosDataset, ChronosBoltDataset, TinyTimeMixerDataset, MoiraiDataset, LPTMDataset
+from samay.utils import load_args, get_gifteval_datasets, get_monash_datasets
 from samay.metric import *
 
 
@@ -29,46 +31,42 @@
 #     "restaurant": ['D'],
 # }
 
-start = time.time()
-NAMES, filesizes = get_gifteval_datasets("data/gifteval")
-end = time.time()
-
-print(f"Time taken to load datasets: {end-start:.2f} seconds")
+SERIES = "monash"
 
-MODEL_NAMES = ["moirai", "chronos", "chronosbolt", "timesfm", "moment", "ttm"]
+MODEL_NAMES = ["moirai", "chronos", "chronosbolt", "timesfm", "moment", "ttm", "lptm"]
 MONASH_NAMES = {
     # "weather": "1D",
-    "tourism_yearly": ["1YE"],
-    "tourism_quarterly": ["1Q"],
-    "tourism_monthly": ["1M"],
-    "cif_2016": ["1M"],
+    "tourism_yearly": "1YE",
+    "tourism_quarterly": "1Q",
+    "tourism_monthly": "1M",
+    "cif_2016": "1M",
     # "london_smart_meters": ["30min"],
-    "australian_electricity_demand": ["30min"],
+    "australian_electricity_demand": "30min",
     # "wind_farms_minutely": ["1min"],
-    "bitcoin": ["1D"],
-    "pedestrian_counts": ["1h"],
-    "vehicle_trips": ["1D"],
-    "kdd_cup_2018": ["1H"],
-    "nn5_daily": ["1D"],
-    "nn5_weekly": ["1W"],
+    "bitcoin": "1D",
+    "pedestrian_counts": "1h",
+    "vehicle_trips": "1D",
+    "kdd_cup_2018": "1H",
+    "nn5_daily": "1D",
+    "nn5_weekly": "1W",
     # "kaggle_web_traffic": ["1D"],
     # "kaggle_web_traffic_weekly": ["1W"],
-    "solar_10_minutes": ["10min"],
-    "solar_weekly": ["1W"],
-    "car_parts": ["1M"],
-    "fred_md": ["1M"],
-    "traffic_hourly": ["1h"],
-    "traffic_weekly": ["1W"],
-    "hospital": ["1M"],
-    "covid_deaths": ["1D"],
-    "sunspot": ["1D"],
-    "saugeenday": ["1D"],
-    "us_births": ["1D"],
-    "solar_4_seconds": ["4s"],
-    "wind_4_seconds": ["4s"],
-    "rideshare": ["1h"],
-    "oikolab_weather": ["1h"],
-    "temperature_rain": ["1D"]
+    "solar_10_minutes": "10min",
+    "solar_weekly": "1W",
+    "car_parts": "1M",
+    "fred_md": "1M",
+    "traffic_hourly": "1h",
+    "traffic_weekly": "1W",
+    "hospital": "1M",
+    "covid_deaths": "1D",
+    "sunspot": "1D",
+    "saugeenday": "1D",
+    "us_births": "1D",
+    "solar_4_seconds": "4s",
+    "wind_4_seconds": "4s",
+    "rideshare": "1h",
+    "oikolab_weather": "1h",
+    "temperature_rain": "1D"
 }
 
 MONASH_SETTINGS = {
@@ -112,6 +110,19 @@
     "chronos": 512
 }
 
+start = time.time()
+if SERIES == "gifteval":
+    # Load the datasets from the Gifteval dataset
+    NAMES = get_gifteval_datasets("data/gifteval")
+elif SERIES == "monash":
+    # Load the datasets from the Monash dataset
+    NAMES = get_monash_datasets("data/monash", MONASH_NAMES, MONASH_SETTINGS)
+
+end = time.time()
+print(NAMES)
+
+print(f"Time taken to load datasets: {end-start:.2f} seconds")
+
 
 def calc_pred_and_context_len(freq):
     # split feq into base and multiplier
@@ -146,10 +157,12 @@ def calc_pred_and_context_len(freq):
 
 if __name__ == "__main__":
 
-    for model_name in ["ttm"]:
+    for model_name in ["moment"]:
         print(f"Evaluating model: {model_name}")
         # create csv file for leaderboard if not already created
         csv_path = f"leaderboard/{model_name}.csv"
+        if SERIES == "monash":
+            csv_path = f"leaderboard/monash_{model_name}.csv"
         if not os.path.exists(csv_path):
             print(f"Creating leaderboard csv file: {csv_path}")
             df = pd.DataFrame(columns=["dataset", "size_in_MB", "eval_time", "mse", "mae", "mase", "mape", "rmse", "nrmse", "smape", "msis", "nd", "mwsq", "crps"])
@@ -171,13 +184,27 @@ def calc_pred_and_context_len(freq):
         elif model_name == "moirai":
             arg_path = "config/moirai.json"
             args = load_args(arg_path)
+        elif model_name == "lptm":
+            arg_path = "config/lptm.json"
+            args = load_args(arg_path)
 
-        for fname, freq, fs in filesizes:
+        for fpath, attrs in NAMES.items():
+            if SERIES == "monash":
+                freq = attrs[0]
+                horizon = attrs[1]
+                fs = attrs[2]
+            elif SERIES == "gifteval":
+                freq = attrs[0]
+                fs = attrs[1]
+            fname = fpath.split("/")[2]
             print(f"Evaluating {fname} ({freq})")
             # Adjust the context and prediction length based on the frequency
 
             # pred_len, context_len = calc_pred_and_context_len(freq)
             pred_len, context_len = 96, 512
+            if SERIES == "monash":
+                pred_len = horizon
+
             if model_name == "timesfm":
                 args["config"]["horizon_len"] = pred_len
                 args["config"]["context_len"] = context_len
@@ -190,12 +217,8 @@ def calc_pred_and_context_len(freq):
                 args["config"]["horizon_len"] = pred_len
                 args["config"]["context_len"] = context_len
 
-            # Set the dataset path
-            if len(NAMES.get(fname)) == 1:
-                dataset_path = f"data/gifteval/{fname}/data.csv"
-            else:
-                dataset_path = f"data/gifteval/{fname}/{freq}/data.csv"
-
+            dataset_path = fpath
+
             if model_name == "timesfm":
 
                 dataset = TimesfmDataset(datetime_col='timestamp', path=dataset_path, mode='test', context_len=args["config"]["context_len"], horizon_len=args["config"]["horizon_len"], boundaries=(-1, -1, -1), batchsize=64)
@@ -208,6 +231,10 @@ def calc_pred_and_context_len(freq):
                 print(f"Size of dataset: {fs:.2f} MB")
                 print(f"Time taken for evaluation of {fname}: {end-start:.2f} seconds")
 
+                del model
+                torch.cuda.empty_cache()
+                gc.collect()
+
             elif model_name == "moment":
 
                 args["config"]["task_name"] = "forecasting"
@@ -223,6 +250,13 @@ def calc_pred_and_context_len(freq):
                 print(f"Time taken for evaluation of {fname}: {end-start:.2f} seconds")
                 print(metrics)
 
+                del model
+                del finetuned_model
+                del dataset
+                del train_dataset
+                torch.cuda.empty_cache()
+                gc.collect()
+
             elif model_name == "chronos":
 
                 dataset_config = load_args("config/chronos_dataset.json")
@@ -237,6 +271,11 @@ def calc_pred_and_context_len(freq):
                 print(f"Size of dataset: {fs:.2f} MB")
                 print(f"Time taken for evaluation of {fname}: {end-start:.2f} seconds")
 
+                del model
+                del dataset
+                torch.cuda.empty_cache()
+                gc.collect()
+
             elif model_name == "chronosbolt":
                 repo = "amazon/chronos-bolt-small"
                 model = ChronosBoltModel(repo=repo)
@@ -247,6 +286,11 @@ def calc_pred_and_context_len(freq):
                 print(f"Size of dataset: {fs:.2f} MB")
                 print(f"Time taken for evaluation of {fname}: {end-start:.2f} seconds")
 
+                del model
+                del dataset
+                torch.cuda.empty_cache()
+                gc.collect()
+
             elif model_name == "ttm":
 
                 dataset = TinyTimeMixerDataset(datetime_col='timestamp', path=dataset_path, mode='test', context_len=context_len, horizon_len=pred_len, boundaries=[-1, -1, -1])
@@ -255,8 +299,14 @@ def calc_pred_and_context_len(freq):
                 start = time.time()
                 metrics = model.evaluate(dataset)
                 end = time.time()
+                print("Metrics: ", metrics)
                 print(f"Size of dataset: {fs:.2f} MB")
                 print(f"Time taken for evaluation of {fname}: {end-start:.2f} seconds")
+
+                del model
+                del dataset
+                torch.cuda.empty_cache()
+                gc.collect()
 
             elif model_name == "moirai":
                 model = MoiraiTSModel(**args)
@@ -268,6 +318,28 @@ def calc_pred_and_context_len(freq):
                 end = time.time()
                 print(f"Size of dataset: {fs:.2f} MB")
                 print(f"Time taken for evaluation of {fname}: {end-start:.2f} seconds")
+
+                del model
+                del dataset
+                torch.cuda.empty_cache()
+                gc.collect()
+
+            elif model_name == "lptm":
+                args["config"]["task_name"] = "forecasting2"
+                dataset = LPTMDataset(name=fname, datetime_col='timestamp', task_name="forecasting2",
+                                        path=dataset_path, mode='test', seq_len=context_len, horizon=pred_len)
+                args["config"]["forecast_horizon"] = dataset.forecast_horizon
+                model = LPTMModel(**args)
+                start = time.time()
+                metrics = model.evaluate(dataset, task_name="forecasting2")
+                end = time.time()
+                print(f"Size of dataset: {fs:.2f} MB")
+                print(f"Time taken for evaluation of {fname}: {end-start:.2f} seconds")
+
+                del model
+                del dataset
+                torch.cuda.empty_cache()
+                gc.collect()
 
             print("Evaluation done!")