Dev

.dockerignore

@@ -1,3 +1,4 @@
 fly.toml
 .git/
 *.sqlite3
+.local/

api/views.py

@@ -8,13 +8,20 @@
 from .serializers import PriceForecastSerializer, PriceForecastRegionSerializer
 
 
-class PriceForecastAPIView(generics.ListAPIView):
-    ids = [f.id for f in Forecasts.objects.all().order_by("-created_at")[:1]]
+# class PriceForecastAPIView(generics.ListAPIView):
+#     ids = [f.id for f in Forecasts.objects.all().order_by("-created_at")[:1]]
+
 
-    queryset = Forecasts.objects.filter(id__in=ids)
-    # queryset = Forecasts.objects.all()
+#     queryset = Forecasts.objects.filter(id__in=ids)
+#     # queryset = Forecasts.objects.all()
+#     serializer_class = PriceForecastSerializer
+class PriceForecastAPIView(generics.ListAPIView):
     serializer_class = PriceForecastSerializer
 
+    def get_queryset(self):
+        latest = Forecasts.objects.order_by("-created_at")[:1]
+        return latest
+
 
 class PriceForecastRegionAPIView(generics.ListAPIView):
     serializer_class = PriceForecastRegionSerializer

config/utils.py

@@ -302,6 +302,7 @@ def get_latest_forecast():
         data, e = DataSet(**x).download()
         if len(data) > 0:
             downloaded_data += [data]
+            # print(f"{x}:\n{data}\n\n")
         else:
             download_errors += [e]
 
@@ -389,7 +390,7 @@ def download(self, tz="GB", params={}):
             pass
 
         try:
-            df.index += pd.to_datetime(df[self.time_col], format="%H:%M") - pd.Timestamp("1900-01-01")
+            df.index += pd.to_datetime(df[self.time_col], format="%H:%M:%S") - pd.Timestamp("1900-01-01")
         except:
             pass
 

prices/forms.py

@@ -5,16 +5,16 @@
 from config.settings import GLOBAL_SETTINGS
 from .models import Forecasts
 
-# REGION_CHOICES = [(r, GLOBAL_SETTINGS["REGIONS"][r]["name"]) for r in GLOBAL_SETTINGS["REGIONS"]]
-forecast_choices = [(f.id, f.name) for f in Forecasts.objects.all().order_by("-created_at")][:14]
-
 
 class ForecastForm(forms.Form):
     forecasts_to_plot = forms.MultipleChoiceField(
-        initial=forecast_choices[0], widget=forms.CheckboxSelectMultiple, choices=forecast_choices
+        widget=forms.CheckboxSelectMultiple,
+        choices=[],  # will be set in __init__
     )
     days_to_plot = forms.ChoiceField(
-        initial=("7", "7"), choices=[(f"{i}", f"{i}") for i in range(1, 14)], required=False
+        initial=("7", "7"),
+        choices=[(f"{i}", f"{i}") for i in range(1, 14)],
+        required=False,
     )
     show_generation_and_demand = forms.BooleanField(
         initial=True,
@@ -33,19 +33,23 @@ class ForecastForm(forms.Form):
     )
 
     def __init__(self, *args, **kwargs):
-        super(ForecastForm, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
+
         self.helper = FormHelper()
         self.helper.form_tag = True
-        # self.helper.form_show_labels = False
         self.fields["forecasts_to_plot"].label = False
+
+        # 🛠️ Move forecast_choices query here
+        forecast_choices = [(f.id, f.name) for f in Forecasts.objects.all().order_by("-created_at")[:14]]
+        self.fields["forecasts_to_plot"].choices = forecast_choices
+        self.fields["forecasts_to_plot"].initial = forecast_choices[0] if forecast_choices else None
+
         self.helper.layout = Layout(
             Accordion(
                 AccordionGroup(
                     "Options",
                     Field("days_to_plot", small=True),
-                    Field(
-                        "show_generation_and_demand",
-                    ),
+                    Field("show_generation_and_demand"),
                     Field("show_range_on_most_recent_forecast"),
                     Field(
                         "show_forecast_overlap",
@@ -61,4 +65,3 @@ def __init__(self, *args, **kwargs):
             ),
             Submit("submit", "Update Chart", css_class="btn btn-light"),
         )
-        print(f">>>base_fields {self.base_fields}")

prices/management/commands/clean_forecasts.py

@@ -16,38 +16,41 @@ def add_arguments(self, parser):
             help="Only keep one forecast per day ",
         )
 
+        parser.add_argument(
+            "--min_fd",
+        )
+
+        parser.add_argument(
+            "--min_ad",
+        )
+
+        parser.add_argument(
+            "--days",
+        )
+
     def handle(self, *args, **options):
         delete = options.get("delete", False)
-        forecast_days = {}
-        for f in Forecasts.objects.all().order_by("-created_at"):
-            q = ForecastData.objects.filter(forecast=f)
-            a = AgileData.objects.filter(forecast=f)
+        min_fd = int(options.get("min_fd", 600) or 600)
+        min_ad = int(options.get("min_ad", 0) or 0)
+        max_days = int(options.get("days", 100000) or 10000)
 
-            print(f.id, f.name, q.count(), a.count())
-            if q.count() < 600 or a.count() < 8000:
-                f.delete()
-            else:
-                if f.created_at.date() in forecast_days:
-                    forecast_days[f.created_at.date()].append(f)
-                else:
-                    forecast_days[f.created_at.date()] = [f]
-
-        print(forecast_days)
+        print(f"Max days: {max_days}")
 
+        print(f"  ID  |       Name       |  #FD  |   #AD   | Days |")
+        print(f"------+------------------+-------+---------+------+")
         keep = []
-        for d in forecast_days:
-            if (pd.Timestamp.now() - pd.Timestamp(d)).days <= 90:
-                # print(d)
-                t = [f.created_at for f in forecast_days[d]]
-                id = [f.id for f in forecast_days[d]]
-                df = pd.Series(index=t, data=id)
-                df.index = df.index.tz_convert("GB")
-
-                z = df[df.index.hour >= 9]
-                if len(z) > 0:
-                    keep.append(z.sort_index().iloc[0])
-                else:
-                    keep.append(df.sort_index().iloc[-1])
+        for f in Forecasts.objects.all().order_by("-created_at"):
+            fd = ForecastData.objects.filter(forecast=f)
+            ad = AgileData.objects.filter(forecast=f)
+            dt = pd.to_datetime(f.name).tz_localize("GB")
+            days = (pd.Timestamp.now(tz="GB") - dt).days
+            if fd.count() < min_fd or ad.count() < min_ad:
+                fail = " <- Fail"
+            else:
+                fail = ""
+                if days < max_days:
+                    keep.append(f.id)
+            print(f"{f.id:5d} | {f.name} | {fd.count():5d} | {ad.count():7d} | {days:4d} | {fail}")
 
         print(keep)
 

prices/management/commands/export_local.py

@@ -7,10 +7,10 @@
 
 class Command(BaseCommand):
     def handle(self, *args, **options):
-        local_dir = os.path.join(os.getcwd(), ".local")
+        local_dir = os.path.join(os.getcwd(), "temp")
         hdf = os.path.join(local_dir, "forecast.hdf")
         if not os.path.exists(local_dir):
-            os.mkdir(".local")
+            os.mkdir("temp")
 
         elif os.path.exists(hdf):
             os.remove(hdf)
@@ -19,12 +19,9 @@ def handle(self, *args, **options):
             key = data.__name__
             print(key)
 
+            df = pd.DataFrame(list(data.objects.all().values()))
             if key == "AgileData":
-                ff = list(pd.DataFrame(list(Forecasts.objects.all().values())).sort_values("created_at")["id"][-10:])
-                print(ff)
-                df = pd.DataFrame(list(data.objects.filter(forecast_id__in=ff).values()))
-            else:
-                df = pd.DataFrame(list(data.objects.all().values()))
+                df = df[df["region"].isin(["G", "X"])]
 
             try:
                 df.to_hdf(hdf, key=key, mode="a")