Merge pull request #39 from bcgov/feat/37-update-fwx-pipe

Feat/37 update fwx pipe

.github/workflows/pr-close.yaml

@@ -1,10 +1,11 @@
 name: Pull Request Closed
 
-on:
-  pull_request:
-    types:
-      - closed
-  workflow_dispatch:
+# disable this action as going to run stuff as gha
+# on:
+#   pull_request:
+#     types:
+#       - closed
+#   workflow_dispatch:
 
 jobs:
   # cleanup should run even if the pr is not merged.

.github/workflows/pr-open-r.yaml

@@ -7,26 +7,29 @@ on:
   # push:
   #   branches: ['feat/23-oc-fire-weather']
 
-  pull_request:
-    # debug: remove the dev,putting build action on non existent branches for debug
-    branches: ['main']
-    types: ['opened', 'reopened', 'edited', 'synchronize']
-    paths-ignore:
-      - .vscode/**
-      - .flake8
-      - .lintr
-      - .RData
-      - .Rhistory
-      - .Rprofile
-      - .gitignore
-      - LICENSE
-      - readme.md
-      - code_of_conduct.md
-      - .flake8
-      - COMPLIANCE.yaml
-      - docs/**
-      - renv.lock
-  workflow_dispatch:
+
+  # disable the build as its not required anymore... Jobs are going to run as
+  # gha.
+  # pull_request:
+  #   # debug: remove the dev,putting build action on non existent branches for debug
+  #   branches: ['main']
+  #   types: ['opened', 'reopened', 'edited', 'synchronize']
+  #   paths-ignore:
+  #     - .vscode/**
+  #     - .flake8
+  #     - .lintr
+  #     - .RData
+  #     - .Rhistory
+  #     - .Rprofile
+  #     - .gitignore
+  #     - LICENSE
+  #     - readme.md
+  #     - code_of_conduct.md
+  #     - .flake8
+  #     - COMPLIANCE.yaml
+  #     - docs/**
+  #     - renv.lock
+  # workflow_dispatch:
 
 jobs:
   # calculate the image tag / used to tag image and then for the

.gitignore

@@ -66,3 +66,5 @@ old_scripts/**
 data/**
 scripts/python/tests/test_data/**
 
+venv_win/**
+

.vscode/settings.json

@@ -1,6 +1,10 @@
 {
     "[python]": {
-        "editor.defaultFormatter": "ms-python.black-formatter"
+        "editor.defaultFormatter": "ms-python.black-formatter",
+        "editor.codeActionsOnSave": {
+            "source.organizeImports": true
+        },
+        "isort.args":["--profile", "black"],
     },
     "python.testing.pytestArgs": [
         "scripts/python"

pyproject.toml

@@ -0,0 +1,83 @@
+[tool.poetry]
+name = "climate-obs"
+version = "0.1.0"
+description = "scripts used to pull and transform climate observations related data"
+authors = ["Kevin Netherton <kevin.netherton@gov.bc.ca>"]
+license = "apache"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.11,<3.13"
+nr-objstore-util = "^0.10.0"
+requests = "^2.31.0"
+beautifulsoup4 = "^4.12.2"
+python-dotenv = "^1.0.0"
+pandas = "^2.1.1"
+
+
+[tool.poetry.group.dev.dependencies]
+black = "23.9.1"
+ruff = "^0.0.292"
+mypy = "^1.5.1"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.ruff]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["E", "F"]
+ignore = []
+
+# Allow fix for all enabled rules (when `--fix`) is provided.
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+unfixable = []
+
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+    "venv_win",
+    "tmp",
+    "data",
+    "scripts/r",
+    "renv",
+    "old_scripts",
+    "junk",
+    "docs",
+    "cicd",
+    "virtualenvs",
+    "usr/lib"
+]
+
+# Same as Black.
+line-length = 88
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+# Assume Python 3.8
+target-version = "py38"
+
+[tool.ruff.mccabe]
+# Unlike Flake8, default to a complexity level of 10.
+max-complexity = 10

scripts/python/fwx_typedicts.py

@@ -0,0 +1,69 @@
+# auto generated using: https://json2pyi.pages.dev/#Dataclass
+
+from __future__ import annotations
+
+from typing import Any, List, TypedDict, Union
+
+
+class WeatherStation(TypedDict):
+    type: str
+    elevation: float
+    fineFuelMoistureCode: float
+    fireCentre: str
+    fireWeatherIndex: float
+    geometry: Geometry
+    initialSpreadIndex: float
+    links: List[Any]
+    precipPluvio1Status: float
+    precipPluvio1Total: float
+    precipPluvio2Status: float
+    precipPluvio2Total: float
+    precipRitTotalPrecipRGT: float
+    precipitation: float
+    recordType: str
+    relativeHumidity: float
+    rn1Pluvio1: None
+    rn1Pluviop2: float
+    rn1RitPrecipRitStatus: float
+    snowDepth: float
+    snowDepthQuality: None
+    solarRadiationCM3: None
+    solarRadiationLicor: float
+    stationAcronym: str
+    stationCode: str
+    stationName: str
+    temperature: float
+    weatherTimestamp: str
+    windDirection: float
+    windGust: float
+    windSpeed: float
+
+class Geometry(TypedDict):
+    coordinates: List[float]
+    crs: Crs
+    type: str
+
+class Crs(TypedDict):
+    properties: Properties
+    type: str
+
+class Properties(TypedDict):
+    name: str
+
+
+class Station(TypedDict):
+    type: str
+    aspect: Union[str, None]
+    elevation: float
+    fireCentre: str
+    fireZone: str
+    geometry: Geometry
+    latitude: float
+    links: List[Any]
+    longitude: float
+    overwinterPrecipInd: bool
+    pluvioSnowGaugeInd: bool
+    slope: Union[None, float]
+    stationAcronym: Union[str, None]
+    stationCode: str
+    stationName: str

scripts/python/logging.config

@@ -1,5 +1,5 @@
 [loggers]
-keys=root,remote_ostore_sync,main_asp
+keys=root,remote_ostore_sync,main_asp,main_win32_climateobs_prep,main_fwx_api
 
 [handlers]
 keys=consoleHandler
@@ -25,6 +25,17 @@ handlers=consoleHandler
 qualname=main_asp
 propagate=0
 
+[logger_main_win32_climateobs_prep]
+level=DEBUG
+handlers=consoleHandler
+qualname=main_win32_climateobs_prep
+propagate=0
+
+[logger_main_fwx_api]
+level=DEBUG
+handlers=consoleHandler
+qualname=main_fwx_api
+propagate=0
 
 
 #----- HANDLERS

scripts/python/main_fwx.py

@@ -2,42 +2,52 @@
 governments data mart.
 """
 
-import remote_ostore_sync
-import os
 import datetime
 import logging.config
+import os
+
+import main_fwx_api
+import remote_ostore_sync
 
+# setup the logging
 cur_path = os.path.dirname(__file__)
 log_config_path = os.path.join(cur_path, 'logging.config')
 
 logging.config.fileConfig(log_config_path, disable_existing_loggers=False)
 LOGGER = logging.getLogger(__name__)
 LOGGER.info(f"starting {__name__}")
 
-default_days_from_present = int(os.getenv('DEFAULT_DAYS_FROM_PRESENT', 1))
+# setting up the time window for pulling data
+end_date = datetime.datetime.now()
+end_date = end_date.replace(
+                hour=9,
+                minute=0,
+                second=0,
+                microsecond=0)
+
+# setting up the path to the output data for the local data
+local_file_date_fmt = '%Y-%m-%d'
+date_str = end_date.strftime(local_file_date_fmt)
+local_file_path_root = os.getenv('F_WX_DATA_DIR', f'./data/fwx')
+local_path = os.path.join(local_file_path_root, 'extracts', date_str)
+if not os.path.exists(local_path):
+    os.makedirs(local_path)
+local_file_path = os.path.join(local_path, f'{date_str}.csv')
 
-current_date = datetime.datetime.now() - datetime.timedelta(days=default_days_from_present)
-yesterday_date = datetime.datetime.now() - datetime.timedelta(days=default_days_from_present + 1)
-current_date_str = current_date.strftime('%Y%m%d')
-yesterday_date_str = yesterday_date.strftime('%Y%m%d')
+# ostore paths
+ostore_data_dir = 'F_WX/extracts'
+ostore_dir_date_fmt = '%Y-%m-%d'
+ostore_path = os.path.join(ostore_data_dir, date_str)
+ostore_file_path = os.path.join(ostore_path, f'{date_str}.csv')
 
-# config local file path
-local_file_date_fmt = '%Y%m%d'
-date_str = current_date.strftime(local_file_date_fmt)
-local_file_path_root = os.getenv('F_WX_DATA_DIR', f'./data/fwx')
-local_file_path = os.path.join(local_file_path_root, 'raw')
-if not os.path.exists(local_file_path):
-    os.makedirs(local_file_path)
-
-cnfig = remote_ostore_sync.SyncRemoteConfig(
-    remote_date_fmt='%Y%m%d',
-    remote_location='http://hpfx.collab.science.gc.ca/{date_str}/WXO-DD/observations/swob-ml/partners/bc-forestry/{date_str}/',
-    ostore_data_dir='F_WX/raw',
-    ostore_dir_date_fmt='%Y%m%d',
-    local_file_path=local_file_path,
-    local_file_date_fmt='%Y%m%d',
-    current_date=current_date
-)
-
-sync = remote_ostore_sync.SyncRemote(cnfig)
-sync.sync()
+# sync
+ostr_sync = remote_ostore_sync.PushProcessed(src_dir=local_path, ostore_dir=ostore_path)
+
+# don't bother doing anything if the data already exists in object storage
+if not ostr_sync.ostore_file_exists(ostore_file_path):
+
+    # now get the data and store remotely
+    fwx_api = main_fwx_api.WildfireAPI(end_date=end_date)
+    fwx_api.get_all_stations_hourlies(local_file_path)
+
+    ostr_sync.sync()