From fbb5a37390105d69d61d2699a5d4c22567e29c0d Mon Sep 17 00:00:00 2001 From: Masnan-306 Date: Mon, 24 Apr 2023 00:57:53 -0400 Subject: [PATCH 01/12] Added sorting feature and Removed Obsolete sub_versions --- web/pandas_web.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/web/pandas_web.py b/web/pandas_web.py index 5e902f1b1919b..b7ef4e1f7b88f 100755 --- a/web/pandas_web.py +++ b/web/pandas_web.py @@ -42,6 +42,7 @@ import markdown import requests import yaml +from packaging import version api_token = os.environ.get("GITHUB_TOKEN") if api_token is not None: @@ -223,7 +224,19 @@ def home_add_releases(context): with open(pathlib.Path(context["target_path"]) / "releases.json", "w") as f: json.dump(releases, f, default=datetime.datetime.isoformat) - for release in releases: + # Sort the releases in descending order + sorted_releases = sorted(releases, key=lambda release:version.parse(release["tag_name"]), reverse=True) + + # Gathers minor versions + latest_releases = [] + minor_versions = set() + for release in sorted_releases: + minor_version = ".".join(release["tag_name"].split(".")[:2]) + if minor_version not in minor_versions: + latest_releases.append(release) + minor_versions.add(minor_version) + + for release in latest_releases: if release["prerelease"]: continue published = datetime.datetime.strptime( From 1a5140f1f89f01b65e3cd4b5f26d779fe92b230f Mon Sep 17 00:00:00 2001 From: Masnan-306 Date: Mon, 24 Apr 2023 01:13:44 -0400 Subject: [PATCH 02/12] Passed CI Pre-commit check --- web/pandas_web.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/web/pandas_web.py b/web/pandas_web.py index b7ef4e1f7b88f..7d0cfc579a698 100755 --- a/web/pandas_web.py +++ b/web/pandas_web.py @@ -40,9 +40,9 @@ import feedparser import jinja2 import markdown +from packaging import version import requests import yaml -from packaging import version api_token = os.environ.get("GITHUB_TOKEN") if api_token is not None: @@ -225,7 +225,11 @@ def home_add_releases(context): json.dump(releases, f, default=datetime.datetime.isoformat) # Sort the releases in descending order - sorted_releases = sorted(releases, key=lambda release:version.parse(release["tag_name"]), reverse=True) + sorted_releases = sorted( + releases, + key=lambda release: version.parse(release["tag_name"]), + reverse=True, + ) # Gathers minor versions latest_releases = [] From d98c1c0626ae8ee7ba4fe2f45b31525eea9f2be1 Mon Sep 17 00:00:00 2001 From: Masnan-306 Date: Fri, 28 Apr 2023 23:02:14 -0400 Subject: [PATCH 03/12] Changed the sorting method with Map --- web/pandas_web.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/web/pandas_web.py b/web/pandas_web.py index 7d0cfc579a698..7134141e162f6 100755 --- a/web/pandas_web.py +++ b/web/pandas_web.py @@ -224,21 +224,21 @@ def home_add_releases(context): with open(pathlib.Path(context["target_path"]) / "releases.json", "w") as f: json.dump(releases, f, default=datetime.datetime.isoformat) + # Map to create a list of (version, realse) pairs from releases + parsed_releases = map(lambda r: (version.parse(r["tag_name"]), r), releases) + # Sort the releases in descending order - sorted_releases = sorted( - releases, - key=lambda release: version.parse(release["tag_name"]), - reverse=True, - ) + sorted_releases = sorted(parsed_releases, reverse=True) - # Gathers minor versions + # Gathers minor versions that are NOT obsolete latest_releases = [] - minor_versions = set() - for release in sorted_releases: - minor_version = ".".join(release["tag_name"].split(".")[:2]) - if minor_version not in minor_versions: - latest_releases.append(release) - minor_versions.add(minor_version) + prev_version = version.Version("0.0.0") + for v, release in sorted_releases: + if (v.major, v.minor) == (prev_version.major, prev_version.minor): + # This release is of obsolete version, so skip + continue + latest_releases.append(release) + prev_version = v for release in latest_releases: if release["prerelease"]: From 25a6eb4dbd6cfc6f59653daf06297ab9c8055756 Mon Sep 17 00:00:00 2001 From: Masnan-306 Date: Sat, 29 Apr 2023 20:33:53 -0400 Subject: [PATCH 04/12] Added tests for sorting releases of pandas based on version --- scripts/tests/test_pandas_web.py | 65 ++++++++++++++++++++++++++++++++ web/pyproject.toml | 1 + web/tests/test_pandas_web.py | 65 ++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 scripts/tests/test_pandas_web.py create mode 100644 web/pyproject.toml create mode 100644 web/tests/test_pandas_web.py diff --git a/scripts/tests/test_pandas_web.py b/scripts/tests/test_pandas_web.py new file mode 100644 index 0000000000000..449cad9c597db --- /dev/null +++ b/scripts/tests/test_pandas_web.py @@ -0,0 +1,65 @@ +from unittest.mock import ( + MagicMock, + patch, +) + +from web.pandas_web import Preprocessors + + +def test_home_add_releases(): + # Prepare test data + releases = [ + { + "tag_name": "v1.0.0", + "prerelease": False, + "published_at": "2021-04-01T00:00:00Z", + "assets": [{"browser_download_url": "https://example.com/download"}], + }, + { + "tag_name": "v2.0.0", + "prerelease": False, + "published_at": "2022-01-01T00:00:00Z", + "assets": [{"browser_download_url": "https://example.com/download"}], + }, + { + "tag_name": "v1.5.4", + "prerelease": False, + "published_at": "2023-08-01T00:00:00Z", + "assets": [], + }, + { + "tag_name": "v1.5.3", + "prerelease": False, + "published_at": "2021-07-01T00:00:00Z", + "assets": [], + }, + { + "tag_name": "v1.5.2", + "prerelease": False, + "published_at": "2021-06-01T00:00:00Z", + "assets": [], + }, + ] + github_repo_url = "pandas-dev/pandas" + context = { + "main": { + "github_repo_url": github_repo_url, + "production_url": "https://example.com/", + }, + "target_path": "/tmp", + "releases": [], + } + + # Mock the requests module to return the test data + resp = MagicMock() + resp.status_code = 200 + resp.json.return_value = releases + with patch("requests.get", return_value=resp): + # Call the function being tested + Preprocessors.home_add_releases(context) + + # Assert that releases were correctly added to the context + assert len(context["releases"]) == 3 + assert context["releases"][0]["name"] == "2.0.0" + assert context["releases"][1]["name"] == "1.5.4" + assert context["releases"][2]["name"] == "1.0.0" diff --git a/web/pyproject.toml b/web/pyproject.toml new file mode 100644 index 0000000000000..8b0bb2780910a --- /dev/null +++ b/web/pyproject.toml @@ -0,0 +1 @@ +[tool.pytest.ini_options] diff --git a/web/tests/test_pandas_web.py b/web/tests/test_pandas_web.py new file mode 100644 index 0000000000000..449cad9c597db --- /dev/null +++ b/web/tests/test_pandas_web.py @@ -0,0 +1,65 @@ +from unittest.mock import ( + MagicMock, + patch, +) + +from web.pandas_web import Preprocessors + + +def test_home_add_releases(): + # Prepare test data + releases = [ + { + "tag_name": "v1.0.0", + "prerelease": False, + "published_at": "2021-04-01T00:00:00Z", + "assets": [{"browser_download_url": "https://example.com/download"}], + }, + { + "tag_name": "v2.0.0", + "prerelease": False, + "published_at": "2022-01-01T00:00:00Z", + "assets": [{"browser_download_url": "https://example.com/download"}], + }, + { + "tag_name": "v1.5.4", + "prerelease": False, + "published_at": "2023-08-01T00:00:00Z", + "assets": [], + }, + { + "tag_name": "v1.5.3", + "prerelease": False, + "published_at": "2021-07-01T00:00:00Z", + "assets": [], + }, + { + "tag_name": "v1.5.2", + "prerelease": False, + "published_at": "2021-06-01T00:00:00Z", + "assets": [], + }, + ] + github_repo_url = "pandas-dev/pandas" + context = { + "main": { + "github_repo_url": github_repo_url, + "production_url": "https://example.com/", + }, + "target_path": "/tmp", + "releases": [], + } + + # Mock the requests module to return the test data + resp = MagicMock() + resp.status_code = 200 + resp.json.return_value = releases + with patch("requests.get", return_value=resp): + # Call the function being tested + Preprocessors.home_add_releases(context) + + # Assert that releases were correctly added to the context + assert len(context["releases"]) == 3 + assert context["releases"][0]["name"] == "2.0.0" + assert context["releases"][1]["name"] == "1.5.4" + assert context["releases"][2]["name"] == "1.0.0" From a1eb2a97c3beceab5eab5421a051c77509ce3443 Mon Sep 17 00:00:00 2001 From: Frank Date: Mon, 1 May 2023 22:16:57 -0400 Subject: [PATCH 05/12] Added comment explaining obselete versions --- web/pandas_web.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/web/pandas_web.py b/web/pandas_web.py index 7134141e162f6..40a2b1ff40ba5 100755 --- a/web/pandas_web.py +++ b/web/pandas_web.py @@ -230,7 +230,9 @@ def home_add_releases(context): # Sort the releases in descending order sorted_releases = sorted(parsed_releases, reverse=True) - # Gathers minor versions that are NOT obsolete + # A version is obsolete if it's not the latest minor version within a major + # version. In the list ["1.4.0", "1.4.1", "1.4.2", "1.5.0", "1.5.1"], versions + # "1.4.0", "1.4.1", and "1.5.0" are obsolete, latest_releases = [] prev_version = version.Version("0.0.0") for v, release in sorted_releases: From c17f2c932dc41693eb1f6c638618c1591bd01b31 Mon Sep 17 00:00:00 2001 From: Frank Date: Mon, 1 May 2023 22:20:38 -0400 Subject: [PATCH 06/12] Integrated sorting into the for loop --- web/pandas_web.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/web/pandas_web.py b/web/pandas_web.py index 40a2b1ff40ba5..cf4eea96ccabc 100755 --- a/web/pandas_web.py +++ b/web/pandas_web.py @@ -227,15 +227,13 @@ def home_add_releases(context): # Map to create a list of (version, realse) pairs from releases parsed_releases = map(lambda r: (version.parse(r["tag_name"]), r), releases) - # Sort the releases in descending order - sorted_releases = sorted(parsed_releases, reverse=True) - # A version is obsolete if it's not the latest minor version within a major # version. In the list ["1.4.0", "1.4.1", "1.4.2", "1.5.0", "1.5.1"], versions # "1.4.0", "1.4.1", and "1.5.0" are obsolete, + # The following code filters out versions that are NOT obsolete latest_releases = [] prev_version = version.Version("0.0.0") - for v, release in sorted_releases: + for v, release in sorted(parsed_releases, reverse=True): if (v.major, v.minor) == (prev_version.major, prev_version.minor): # This release is of obsolete version, so skip continue From 04d89a492f825e546d7614f332d4f7216def23b8 Mon Sep 17 00:00:00 2001 From: Frank Date: Mon, 1 May 2023 22:30:19 -0400 Subject: [PATCH 07/12] deleted duplicate test --- scripts/tests/test_pandas_web.py | 65 -------------------------------- 1 file changed, 65 deletions(-) delete mode 100644 scripts/tests/test_pandas_web.py diff --git a/scripts/tests/test_pandas_web.py b/scripts/tests/test_pandas_web.py deleted file mode 100644 index 449cad9c597db..0000000000000 --- a/scripts/tests/test_pandas_web.py +++ /dev/null @@ -1,65 +0,0 @@ -from unittest.mock import ( - MagicMock, - patch, -) - -from web.pandas_web import Preprocessors - - -def test_home_add_releases(): - # Prepare test data - releases = [ - { - "tag_name": "v1.0.0", - "prerelease": False, - "published_at": "2021-04-01T00:00:00Z", - "assets": [{"browser_download_url": "https://example.com/download"}], - }, - { - "tag_name": "v2.0.0", - "prerelease": False, - "published_at": "2022-01-01T00:00:00Z", - "assets": [{"browser_download_url": "https://example.com/download"}], - }, - { - "tag_name": "v1.5.4", - "prerelease": False, - "published_at": "2023-08-01T00:00:00Z", - "assets": [], - }, - { - "tag_name": "v1.5.3", - "prerelease": False, - "published_at": "2021-07-01T00:00:00Z", - "assets": [], - }, - { - "tag_name": "v1.5.2", - "prerelease": False, - "published_at": "2021-06-01T00:00:00Z", - "assets": [], - }, - ] - github_repo_url = "pandas-dev/pandas" - context = { - "main": { - "github_repo_url": github_repo_url, - "production_url": "https://example.com/", - }, - "target_path": "/tmp", - "releases": [], - } - - # Mock the requests module to return the test data - resp = MagicMock() - resp.status_code = 200 - resp.json.return_value = releases - with patch("requests.get", return_value=resp): - # Call the function being tested - Preprocessors.home_add_releases(context) - - # Assert that releases were correctly added to the context - assert len(context["releases"]) == 3 - assert context["releases"][0]["name"] == "2.0.0" - assert context["releases"][1]["name"] == "1.5.4" - assert context["releases"][2]["name"] == "1.0.0" From 23f0a8a04b4cd6d28a73d1e2dad973a8890cab06 Mon Sep 17 00:00:00 2001 From: Masnan-306 Date: Mon, 1 May 2023 22:35:17 -0400 Subject: [PATCH 08/12] Added comments for example of filtering obsolete releases --- web/pandas_web.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/web/pandas_web.py b/web/pandas_web.py index cf4eea96ccabc..9f1683b293680 100755 --- a/web/pandas_web.py +++ b/web/pandas_web.py @@ -224,13 +224,14 @@ def home_add_releases(context): with open(pathlib.Path(context["target_path"]) / "releases.json", "w") as f: json.dump(releases, f, default=datetime.datetime.isoformat) - # Map to create a list of (version, realse) pairs from releases parsed_releases = map(lambda r: (version.parse(r["tag_name"]), r), releases) - # A version is obsolete if it's not the latest minor version within a major - # version. In the list ["1.4.0", "1.4.1", "1.4.2", "1.5.0", "1.5.1"], versions + # A version is obsolete if it's not the latest minor version. + # In the list ["1.4.0", "1.4.1", "1.4.2", "1.5.0", "1.5.1"], versions # "1.4.0", "1.4.1", and "1.5.0" are obsolete, - # The following code filters out versions that are NOT obsolete + # We don't need 1.5.0 because 1.5.1 has the same features with some bugs fixed + + # Filter obsolete versions latest_releases = [] prev_version = version.Version("0.0.0") for v, release in sorted(parsed_releases, reverse=True): From a4c6aa259a1ef83ef0987146aff492427f1e7ff1 Mon Sep 17 00:00:00 2001 From: Masnan-306 Date: Wed, 31 May 2023 17:08:46 +0800 Subject: [PATCH 09/12] Combine the two for-loops into one for simplicity --- web/pandas_web.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/web/pandas_web.py b/web/pandas_web.py index 9f1683b293680..bf537eb4d0400 100755 --- a/web/pandas_web.py +++ b/web/pandas_web.py @@ -224,24 +224,26 @@ def home_add_releases(context): with open(pathlib.Path(context["target_path"]) / "releases.json", "w") as f: json.dump(releases, f, default=datetime.datetime.isoformat) - parsed_releases = map(lambda r: (version.parse(r["tag_name"]), r), releases) + parsed_releases = map( + lambda release: (version.parse(release["tag_name"]), release), releases + ) + # The following filters out obsolete releases # A version is obsolete if it's not the latest minor version. # In the list ["1.4.0", "1.4.1", "1.4.2", "1.5.0", "1.5.1"], versions # "1.4.0", "1.4.1", and "1.5.0" are obsolete, # We don't need 1.5.0 because 1.5.1 has the same features with some bugs fixed - # Filter obsolete versions - latest_releases = [] + # Dummy version for filtering obsolete versions prev_version = version.Version("0.0.0") + for v, release in sorted(parsed_releases, reverse=True): + # Example: If "1.5.3" is already added, we skip "1.5.2" and "1.5.1" + # Only the most up-to-date "1.5.x" release would be included due to sorting if (v.major, v.minor) == (prev_version.major, prev_version.minor): - # This release is of obsolete version, so skip continue - latest_releases.append(release) prev_version = v - for release in latest_releases: if release["prerelease"]: continue published = datetime.datetime.strptime( From 84f5969945a64785c7abfd9fdca956afde6d1fe2 Mon Sep 17 00:00:00 2001 From: Masnan-306 Date: Wed, 31 May 2023 18:48:35 +0800 Subject: [PATCH 10/12] Fixed pre-commit error --- web/pandas_web.py | 1 + 1 file changed, 1 insertion(+) diff --git a/web/pandas_web.py b/web/pandas_web.py index bf537eb4d0400..5c2be16a21f48 100755 --- a/web/pandas_web.py +++ b/web/pandas_web.py @@ -242,6 +242,7 @@ def home_add_releases(context): # Only the most up-to-date "1.5.x" release would be included due to sorting if (v.major, v.minor) == (prev_version.major, prev_version.minor): continue + prev_version = v if release["prerelease"]: From 62e9e69619a275005665e96a8d2f1b2d58e87240 Mon Sep 17 00:00:00 2001 From: Masnan-306 Date: Wed, 31 May 2023 18:51:40 +0800 Subject: [PATCH 11/12] Fixed pre-commit error --- web/pandas_web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/pandas_web.py b/web/pandas_web.py index 5c2be16a21f48..f260986f1915e 100755 --- a/web/pandas_web.py +++ b/web/pandas_web.py @@ -232,7 +232,7 @@ def home_add_releases(context): # A version is obsolete if it's not the latest minor version. # In the list ["1.4.0", "1.4.1", "1.4.2", "1.5.0", "1.5.1"], versions # "1.4.0", "1.4.1", and "1.5.0" are obsolete, - # We don't need 1.5.0 because 1.5.1 has the same features with some bugs fixed + # We don't need 1.5.0 as 1.5.1 has the same features with some bugs fixed # Dummy version for filtering obsolete versions prev_version = version.Version("0.0.0") From 1812b25a8a7ae9870cf25c8812a3ec61191b9e7d Mon Sep 17 00:00:00 2001 From: Masnan-306 Date: Wed, 31 May 2023 18:57:18 +0800 Subject: [PATCH 12/12] Fixed pre-commit error --- web/pandas_web.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/pandas_web.py b/web/pandas_web.py index f260986f1915e..1ef37e20342e8 100755 --- a/web/pandas_web.py +++ b/web/pandas_web.py @@ -224,9 +224,9 @@ def home_add_releases(context): with open(pathlib.Path(context["target_path"]) / "releases.json", "w") as f: json.dump(releases, f, default=datetime.datetime.isoformat) - parsed_releases = map( - lambda release: (version.parse(release["tag_name"]), release), releases - ) + parsed_releases = [ + (version.parse(release["tag_name"]), release) for release in releases + ] # The following filters out obsolete releases # A version is obsolete if it's not the latest minor version.