From 7537c7e5b7978e826f46f858d2b9169b4abf1987 Mon Sep 17 00:00:00 2001 From: ltsaprounis Date: Wed, 20 Dec 2023 19:13:15 +0000 Subject: [PATCH 1/6] ruff for linting and formatting and CI with linting step --- .github/workflows/python_ci.yml | 16 +++++++ python/pyproject.toml | 76 ++++++++++++++++++++++++++++++--- 2 files changed, 85 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/python_ci.yml diff --git a/.github/workflows/python_ci.yml b/.github/workflows/python_ci.yml new file mode 100644 index 00000000..51350d38 --- /dev/null +++ b/.github/workflows/python_ci.yml @@ -0,0 +1,16 @@ +name: Python CI + +on: + push: + branches: [Python] + pull_request: + branches: [Python] + +jobs: + linting: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: chartboost/ruff-action@v1 + with: + src: "./python" diff --git a/python/pyproject.toml b/python/pyproject.toml index 0534d7d0..ba20aaf9 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -10,14 +10,8 @@ readme = "README.md" requires-python = ">=3.8,<3.11" dependencies = ["pybind11[global]>=2.6.0", "numpy>=1.14"] -# [tool.setuptools.packages] -# find = {} - [project.optional-dependencies] -dev = ["flake8", "black", "pytest", "pydocstyle", "pre-commit", "ipykernel"] - -# [tool.pytest.ini_options] -# addopts = "--ignore=../src/carma" +dev = ["pytest", "ruff", "pre-commit", "ipykernel"] [tool.scikit-build] # wheel.expand-macos-universal-tags = true @@ -28,3 +22,71 @@ ninja.make-fallback = true cmake.verbose = true ninja.minimum-version = "1.5" cmake.minimum-version = "3.25" + +[tool.ruff] +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".mypy_cache", + ".nox", + ".pants.d", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "venv", + "tools/cookiecutter_templates", +] + +# Same as Black. +line-length = 88 +indent-width = 4 + +# Assume Python 3.8 +target-version = "py38" + +[tool.ruff.lint] +# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. +# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or +# McCabe complexity (`C901`) by default. +select = [ + "E4", + "E7", + "E9", + "F", + "I", # isort rules + "E501", +] +ignore = [] + +# Allow fix for all enabled rules (when `--fix`) is provided. +fixable = ["ALL"] +unfixable = [] + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +[tool.ruff.format] +# Like Black, use double quotes for strings. +quote-style = "double" + +# Like Black, indent with spaces, rather than tabs. +indent-style = "space" + +# Like Black, respect magic trailing commas. +skip-magic-trailing-comma = false + +# Like Black, automatically detect the appropriate line ending. +line-ending = "auto" From 18873e96324656165054b33ffe85784a8a9b8b3e Mon Sep 17 00:00:00 2001 From: ltsaprounis Date: Wed, 20 Dec 2023 21:03:18 +0000 Subject: [PATCH 2/6] remove dummy my_linalg module --- python/smooth/my_linalg/__init__.py | 6 ---- python/smooth/my_linalg/tests/__init__.py | 0 .../smooth/my_linalg/tests/test_my_linalg.py | 33 ------------------- 3 files changed, 39 deletions(-) delete mode 100644 python/smooth/my_linalg/__init__.py delete mode 100644 python/smooth/my_linalg/tests/__init__.py delete mode 100644 python/smooth/my_linalg/tests/test_my_linalg.py diff --git a/python/smooth/my_linalg/__init__.py b/python/smooth/my_linalg/__init__.py deleted file mode 100644 index e7e7ac31..00000000 --- a/python/smooth/my_linalg/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Example python module for using carma and pybind11.""" - -from ._my_linalg import example_attr, add, dot_product, arma_dot_product, array_sum - - -__all__ = ["example_attr", "add", "dot_product", "arma_dot_product", "array_sum"] diff --git a/python/smooth/my_linalg/tests/__init__.py b/python/smooth/my_linalg/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/python/smooth/my_linalg/tests/test_my_linalg.py b/python/smooth/my_linalg/tests/test_my_linalg.py deleted file mode 100644 index 02db4436..00000000 --- a/python/smooth/my_linalg/tests/test_my_linalg.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Tests the my_linalg library.""" -import numpy as np - -from smooth.my_linalg import add, dot_product, array_sum, arma_dot_product - -arr1 = np.array([1, 2]) -arr2 = np.array([3, 4]) - - -def test_add(): - """Test the add function.""" - assert add(1, 2) == 3 - - -def test_dot_product(): - """Test the dot_product function""" - arr1 = np.array([1, 2]) - arr2 = np.array([3, 4]) - assert dot_product(arr1, arr2) == 11 - - -def test_arma_dot_product(): - """Tests the arma_dot_product function""" - arr1 = np.array([1, 2]) - arr2 = np.array([3, 4]) - assert arma_dot_product(arr1, arr2) == 11 - - -def test_array_sum(): - """Tests the array_sum function""" - arr1 = np.array([1, 2]) - arr2 = np.array([3, 4]) - assert np.all(array_sum(arr1, arr2) == np.array([4, 6])) From d96603bbdfe8960597ece59087e031d49f614004 Mon Sep 17 00:00:00 2001 From: ltsaprounis Date: Wed, 20 Dec 2023 21:04:05 +0000 Subject: [PATCH 3/6] pass all ruff checks (not full scope of rules) --- python/smooth/adam_general/adam_profile.py | 17 +++++++++++------ python/smooth/adam_general/sma.py | 9 ++++----- python/smooth/adam_general/test_script.py | 1 + 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/python/smooth/adam_general/adam_profile.py b/python/smooth/adam_general/adam_profile.py index 81f37a93..ef40340c 100644 --- a/python/smooth/adam_general/adam_profile.py +++ b/python/smooth/adam_general/adam_profile.py @@ -14,7 +14,8 @@ def adamProfileCreator( yIndex (list): The indices needed to get the specific dates (optional). yClasses (list): The class used for the actual data (optional). Returns: - dict: A dictionary with 'recent' (profilesRecentTable) and 'lookup' (indexLookupTable) as keys. + dict: A dictionary with 'recent' (profilesRecentTable) and 'lookup' + (indexLookupTable) as keys. """ # Initialize matrices profilesRecentTable = np.zeros((len(lagsModelAll), lagsModelMax)) @@ -31,10 +32,12 @@ def adamProfileCreator( # For every row, fill the first 'lag' elements from 1 to lag profilesRecentTable[i, : lag[0]] = np.arange(1, lag[0] + 1) - # For the i-th row in indexLookupTable, fill with a repeated sequence starting from lagsModelMax to the end of the row. - # The repeated sequence is the i-th row of profileIndices, repeated enough times to cover 'obsAll' observations. + # For the i-th row in indexLookupTable, fill with a repeated sequence starting + # from lagsModelMax to the end of the row. + # The repeated sequence is the i-th row of profileIndices, repeated enough times + # to cover 'obsAll' observations. # '- 1' at the end adjusts these values to Python's zero-based indexing. - indexLookupTable[i, lagsModelMax : (lagsModelMax + obsAll)] = ( # noqa + indexLookupTable[i, lagsModelMax : (lagsModelMax + obsAll)] = ( np.tile( profileIndices[i, : lagsModelAll[i][0]], int(np.ceil(obsAll / lagsModelAll[i][0])), @@ -42,13 +45,15 @@ def adamProfileCreator( - 1 ) - # Extract unique values from from lagsModelMax to lagsModelMax + obsAll of indexLookupTable + # Extract unique values from from lagsModelMax to lagsModelMax + obsAll of + # indexLookupTable unique_values = np.unique( indexLookupTable[i, lagsModelMax : lagsModelMax + obsAll] # noqa ) # fix the head of teh data before the sample starts - # Repeat the unique values lagsModelMax times and then trim the sequence to only keep the first lagsModelMax elements + # Repeat the unique values lagsModelMax times and then trim the sequence to only + # keep the first lagsModelMax elements indexLookupTable[i, :lagsModelMax] = np.tile(unique_values, lagsModelMax)[ -lagsModelMax: ] diff --git a/python/smooth/adam_general/sma.py b/python/smooth/adam_general/sma.py index c37d0c87..7ac42342 100644 --- a/python/smooth/adam_general/sma.py +++ b/python/smooth/adam_general/sma.py @@ -1,4 +1,5 @@ import numpy as np + from smooth.adam_general._adam_general import adam_fitter from smooth.adam_general.adam_profile import adamProfileCreator @@ -7,7 +8,7 @@ def sma(y, order=1, h=10, holdout=False): """SMA""" y = y.astype(np.float64) - ic = lambda e: np.sum(e**2) + # ic = lambda e: np.sum(e**2) obs_all = len(y) + h * (1 - holdout) obs_in_sample = len(y) - h * holdout y_in_sample = y @@ -25,14 +26,12 @@ def sma(y, order=1, h=10, holdout=False): def creator_sma(order): # lags_model_all = np.ones(shape=(order, 1)) # This needs to be a vector of values - lags_model_all = np.arange(1, order+1, dtype="int32").reshape(order, 1) + lags_model_all = np.arange(1, order + 1, dtype="int32").reshape(order, 1) lags_model_max = int(max(lags_model_all)) obs_states = obs_in_sample + lags_model_max profiles_recent_table, index_lookup_table = adamProfileCreator( - lagsModelAll=lags_model_all, - lagsModelMax=lags_model_max, - obsAll=obs_all + lagsModelAll=lags_model_all, lagsModelMax=lags_model_max, obsAll=obs_all ).values() # # This needs to be generated by a profileCreator() function diff --git a/python/smooth/adam_general/test_script.py b/python/smooth/adam_general/test_script.py index 535516c5..7e1e28cd 100644 --- a/python/smooth/adam_general/test_script.py +++ b/python/smooth/adam_general/test_script.py @@ -1,4 +1,5 @@ import numpy as np + from smooth.adam_general.sma import sma if __name__ == "__main__": From 88e9b10f48f421280647d9ec2d5e6f6186228981 Mon Sep 17 00:00:00 2001 From: ltsaprounis Date: Wed, 20 Dec 2023 21:34:14 +0000 Subject: [PATCH 4/6] added N rules in ruff rule scope. --- python/pyproject.toml | 9 +++-- python/smooth/adam_general/adam_profile.py | 38 +++++++++++----------- python/smooth/adam_general/sma.py | 24 +++++++------- 3 files changed, 39 insertions(+), 32 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index ba20aaf9..f32773be 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -65,11 +65,16 @@ select = [ "E4", "E7", "E9", + "E501", "F", "I", # isort rules - "E501", + "N", # PEP-8 naming rules +] +ignore = [ + # uppercase names are used for matrices + "N803", # allow uppercase argument names + "N806", # allow uppercase variable names ] -ignore = [] # Allow fix for all enabled rules (when `--fix`) is provided. fixable = ["ALL"] diff --git a/python/smooth/adam_general/adam_profile.py b/python/smooth/adam_general/adam_profile.py index ef40340c..dbadf8e6 100644 --- a/python/smooth/adam_general/adam_profile.py +++ b/python/smooth/adam_general/adam_profile.py @@ -1,8 +1,8 @@ import numpy as np -def adamProfileCreator( - lagsModelAll, lagsModelMax, obsAll, lags=None, yIndex=None, yClasses=None +def adam_profile_creator( + lags_model_all, lags_model_max, obs_all, lags=None, y_index=None, y_classes=None ): """ Creates recent profile and the lookup table for adam. @@ -18,51 +18,51 @@ def adamProfileCreator( (indexLookupTable) as keys. """ # Initialize matrices - profilesRecentTable = np.zeros((len(lagsModelAll), lagsModelMax)) - indexLookupTable = np.ones((len(lagsModelAll), obsAll + lagsModelMax)) - profileIndices = ( - np.arange(1, lagsModelMax * len(lagsModelAll) + 1) - .reshape(-1, len(lagsModelAll)) + profiles_recent_table = np.zeros((len(lags_model_all), lags_model_max)) + index_lookup_table = np.ones((len(lags_model_all), obs_all + lags_model_max)) + profile_indices = ( + np.arange(1, lags_model_max * len(lags_model_all) + 1) + .reshape(-1, len(lags_model_all)) .T ) # Update matrices based on lagsModelAll - for i, lag in enumerate(lagsModelAll): + for i, lag in enumerate(lags_model_all): # Create the matrix with profiles based on the provided lags. # For every row, fill the first 'lag' elements from 1 to lag - profilesRecentTable[i, : lag[0]] = np.arange(1, lag[0] + 1) + profiles_recent_table[i, : lag[0]] = np.arange(1, lag[0] + 1) # For the i-th row in indexLookupTable, fill with a repeated sequence starting # from lagsModelMax to the end of the row. # The repeated sequence is the i-th row of profileIndices, repeated enough times # to cover 'obsAll' observations. # '- 1' at the end adjusts these values to Python's zero-based indexing. - indexLookupTable[i, lagsModelMax : (lagsModelMax + obsAll)] = ( + index_lookup_table[i, lags_model_max : (lags_model_max + obs_all)] = ( np.tile( - profileIndices[i, : lagsModelAll[i][0]], - int(np.ceil(obsAll / lagsModelAll[i][0])), - )[0:obsAll] + profile_indices[i, : lags_model_all[i][0]], + int(np.ceil(obs_all / lags_model_all[i][0])), + )[0:obs_all] - 1 ) # Extract unique values from from lagsModelMax to lagsModelMax + obsAll of # indexLookupTable unique_values = np.unique( - indexLookupTable[i, lagsModelMax : lagsModelMax + obsAll] # noqa + index_lookup_table[i, lags_model_max : lags_model_max + obs_all] # noqa ) # fix the head of teh data before the sample starts # Repeat the unique values lagsModelMax times and then trim the sequence to only # keep the first lagsModelMax elements - indexLookupTable[i, :lagsModelMax] = np.tile(unique_values, lagsModelMax)[ - -lagsModelMax: + index_lookup_table[i, :lags_model_max] = np.tile(unique_values, lags_model_max)[ + -lags_model_max: ] # Convert to int! - indexLookupTable = indexLookupTable.astype(int) + index_lookup_table = index_lookup_table.astype(int) # Note: I skip andling of special cases (e.g., daylight saving time, leap years) return { - "recent": np.array(profilesRecentTable, dtype="float64"), - "lookup": np.array(indexLookupTable, dtype="int64"), + "recent": np.array(profiles_recent_table, dtype="float64"), + "lookup": np.array(index_lookup_table, dtype="int64"), } diff --git a/python/smooth/adam_general/sma.py b/python/smooth/adam_general/sma.py index 7ac42342..9c44de47 100644 --- a/python/smooth/adam_general/sma.py +++ b/python/smooth/adam_general/sma.py @@ -1,7 +1,7 @@ import numpy as np from smooth.adam_general._adam_general import adam_fitter -from smooth.adam_general.adam_profile import adamProfileCreator +from smooth.adam_general.adam_profile import adam_profile_creator def sma(y, order=1, h=10, holdout=False): @@ -30,8 +30,10 @@ def creator_sma(order): lags_model_max = int(max(lags_model_all)) obs_states = obs_in_sample + lags_model_max - profiles_recent_table, index_lookup_table = adamProfileCreator( - lagsModelAll=lags_model_all, lagsModelMax=lags_model_max, obsAll=obs_all + profiles_recent_table, index_lookup_table = adam_profile_creator( + lags_model_all=lags_model_all, + lags_model_max=lags_model_max, + obs_all=obs_all, ).values() # # This needs to be generated by a profileCreator() function @@ -44,19 +46,19 @@ def creator_sma(order): # np.arange(order), (obs_all + lags_model_max, 1) # ).T - matF = np.ones((order, order)) / order - matWt = np.ones((obs_in_sample, order)) + mat_F = np.ones((order, order)) / order + mat_Wt = np.ones((obs_in_sample, order)) - vecG = np.ones(order) / order + vec_G = np.ones(order) / order # matVt = np.zeros((order, obs_states)) - matVt = np.empty((order, obs_states)) + mat_Vt = np.empty((order, obs_states)) # matVt.fill(np.nan) adam_fitted = adam_fitter( - matrixVt=matVt, - matrixWt=matWt, - matrixF=matF, - vectorG=vecG, + matrixVt=mat_Vt, + matrixWt=mat_Wt, + matrixF=mat_F, + vectorG=vec_G, lags=lags_model_all, indexLookupTable=index_lookup_table, profilesRecent=profiles_recent_table, From d1641dc6be327037e72aaa4b3b5e3a6bb536e338 Mon Sep 17 00:00:00 2001 From: ltsaprounis Date: Mon, 8 Jan 2024 22:34:47 +0000 Subject: [PATCH 5/6] small comment in pyproject.toml --- python/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/pyproject.toml b/python/pyproject.toml index f32773be..9cb90808 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -61,6 +61,7 @@ target-version = "py38" # Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. # Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or # McCabe complexity (`C901`) by default. +# Rules for ruff are found here: https://docs.astral.sh/ruff/rules/ select = [ "E4", "E7", From c0b48b2f9d69a4eb4731e90712eaa2fd4f0a400b Mon Sep 17 00:00:00 2001 From: ltsaprounis Date: Tue, 13 Feb 2024 18:07:27 +0000 Subject: [PATCH 6/6] transfer adam_forecaster --- python/smooth/adam_general/sma.py | 52 +++++++++++++++++++++-- python/smooth/adam_general/test_script.py | 2 +- src/python_examples/adamGeneral.cpp | 43 +++++++++++++++++++ 3 files changed, 93 insertions(+), 4 deletions(-) diff --git a/python/smooth/adam_general/sma.py b/python/smooth/adam_general/sma.py index 9c44de47..7c9fd979 100644 --- a/python/smooth/adam_general/sma.py +++ b/python/smooth/adam_general/sma.py @@ -1,6 +1,6 @@ import numpy as np -from smooth.adam_general._adam_general import adam_fitter +from smooth.adam_general._adam_general import adam_fitter, adam_forecaster from smooth.adam_general.adam_profile import adam_profile_creator @@ -75,6 +75,52 @@ def creator_sma(order): backcast=True, ) - return adam_fitted + fitted_args = dict( + matrixVt=mat_Vt, + matrixWt=mat_Wt, + matrixF=mat_F, + vectorG=vec_G, + lags=lags_model_all, + indexLookupTable=index_lookup_table, + profilesRecent=profiles_recent_table, + E=E_type, + T=T_type, + S=S_type, + nNonSeasonal=components_num_ETS, + nSeasonal=components_num_ETS_seasonal, + nArima=order, + nXreg=xreg_number, + constant=constant_required, + vectorYt=y_in_sample, + vectorOt=ot, + backcast=True, + ) + + return adam_fitted, fitted_args + + sma_fitted, fitted_args = creator_sma(order=order) + + # need to convert some inputs to the expected dtypes. This is a temporary fix. + fitted_args["lags"] = np.array(fitted_args["lags"], dtype="uint64") + fitted_args["indexLookupTable"] = np.array( + fitted_args["indexLookupTable"], dtype="uint64" + ) + + sma_forecast = adam_forecaster( + matrixWt=fitted_args["matrixWt"], + matrixF=fitted_args["matrixF"], + lags=fitted_args["lags"], + indexLookupTable=fitted_args["indexLookupTable"], + profilesRecent=sma_fitted["profile"], + E=fitted_args["E"], + T=fitted_args["T"], + S=fitted_args["S"], + nNonSeasonal=fitted_args["nNonSeasonal"], + nSeasonal=fitted_args["nSeasonal"], + nArima=fitted_args["nArima"], + nXreg=fitted_args["nXreg"], + constant=fitted_args["constant"], + horizon=h, + ) - return creator_sma(order=order) + return sma_forecast diff --git a/python/smooth/adam_general/test_script.py b/python/smooth/adam_general/test_script.py index 7e1e28cd..48bcaa58 100644 --- a/python/smooth/adam_general/test_script.py +++ b/python/smooth/adam_general/test_script.py @@ -5,4 +5,4 @@ if __name__ == "__main__": y = np.arange(0, 100) results = sma(y, order=5) - print(results["yFitted"]) + print(results) diff --git a/src/python_examples/adamGeneral.cpp b/src/python_examples/adamGeneral.cpp index 80d24514..f7569164 100644 --- a/src/python_examples/adamGeneral.cpp +++ b/src/python_examples/adamGeneral.cpp @@ -159,6 +159,34 @@ py::dict adamFitter(arma::mat &matrixVt, arma::mat const &matrixWt, arma::mat &m return result; } +/* # Function produces the point forecasts for the specified model */ +arma::vec adamForecaster(arma::mat const &matrixWt, arma::mat const &matrixF, + arma::uvec lags, arma::umat const &indexLookupTable, arma::mat profilesRecent, + char const &E, char const &T, char const &S, + unsigned int const &nNonSeasonal, unsigned int const &nSeasonal, + unsigned int const &nArima, unsigned int const &nXreg, bool const &constant, + unsigned int const &horizon) +{ + // unsigned int lagslength = lags.n_rows; + unsigned int nETS = nNonSeasonal + nSeasonal; + unsigned int nComponents = indexLookupTable.n_rows; + + arma::vec vecYfor(horizon, arma::fill::zeros); + + /* # Fill in the new xt matrix using F. Do the forecasts. */ + for (unsigned int i = 0; i < horizon; i = i + 1) + { + vecYfor.row(i) = adamWvalue(profilesRecent(indexLookupTable.col(i)), matrixWt.row(i), E, T, S, + nETS, nNonSeasonal, nSeasonal, nArima, nXreg, nComponents, constant); + + profilesRecent(indexLookupTable.col(i)) = adamFvalue(profilesRecent(indexLookupTable.col(i)), + matrixF, E, T, S, nETS, nNonSeasonal, nSeasonal, nArima, nComponents, constant); + } + + // return List::create(Named("matVt") = matrixVtnew, Named("yForecast") = vecYfor); + return vecYfor; +} + PYBIND11_MODULE(_adam_general, m) { m.doc() = "Adam code"; // module docstring @@ -184,4 +212,19 @@ PYBIND11_MODULE(_adam_general, m) py::arg("vectorYt"), py::arg("vectorOt"), py::arg("backcast")); + m.def("adam_forecaster", &adamForecaster, "forecasts the adam model", + py::arg("matrixWt"), + py::arg("matrixF"), + py::arg("lags"), + py::arg("indexLookupTable"), + py::arg("profilesRecent"), + py::arg("E"), + py::arg("T"), + py::arg("S"), + py::arg("nNonSeasonal"), + py::arg("nSeasonal"), + py::arg("nArima"), + py::arg("nXreg"), + py::arg("constant"), + py::arg("horizon")); }