From e9318e328be6b56397a9379553842c30f440c880 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Fri, 30 Jun 2023 09:23:42 -0500 Subject: [PATCH 1/3] Add metrics for projects using OIDC publishers --- tests/unit/oidc/__init__.py | 0 tests/unit/oidc/test_tasks.py | 42 +++++++++++++++++++++++++++++++++++ warehouse/oidc/__init__.py | 6 +++++ warehouse/oidc/tasks.py | 24 ++++++++++++++++++++ 4 files changed, 72 insertions(+) create mode 100644 tests/unit/oidc/__init__.py create mode 100644 tests/unit/oidc/test_tasks.py create mode 100644 warehouse/oidc/tasks.py diff --git a/tests/unit/oidc/__init__.py b/tests/unit/oidc/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/unit/oidc/test_tasks.py b/tests/unit/oidc/test_tasks.py new file mode 100644 index 000000000000..f4427f562fed --- /dev/null +++ b/tests/unit/oidc/test_tasks.py @@ -0,0 +1,42 @@ +import pretend + +from warehouse.oidc.tasks import compute_oidc_metrics + +from ...common.db.oidc import GitHubPublisherFactory +from ...common.db.packaging import ProjectFactory + + +def test_compute_oidc_metrics(db_request, monkeypatch): + # Projects with OIDC + critical_project_oidc = ProjectFactory.create( + name="critical_project_oidc", pypi_mandates_2fa=True + ) + non_critical_project_oidc = ProjectFactory.create( + name="non_critical_project_oidc", + ) + + # Projects without OIDC + critical_project_no_oidc = ProjectFactory.create( + name="critical_project_no_oidc", pypi_mandates_2fa=True + ) + non_critical_project_no_oidc = ProjectFactory.create( + name="non_critical_project_no_oidc", + ) + + # Create OIDC publishers. One OIDCPublisher can be shared by multiple + # projects so 'oidc_publisher_1' represents that situation. Verify + # that metrics don't double-count projects using multiple OIDC publishers. + oidc_publisher_1 = GitHubPublisherFactory.create( + projects=[critical_project_oidc, non_critical_project_oidc] + ) + oidc_publisher_2 = GitHubPublisherFactory.create(projects=[critical_project_oidc]) + + gauge = pretend.call_recorder(lambda metric, value: None) + db_request.find_service = lambda *a, **kw: pretend.stub(gauge=gauge) + + compute_oidc_metrics(db_request) + + assert gauge.calls == [ + pretend.call("warehouse.oidc.total_projects_using_oidc_publishers", 2), + pretend.call("warehouse.oidc.total_critical_projects_using_oidc_publishers", 1), + ] diff --git a/warehouse/oidc/__init__.py b/warehouse/oidc/__init__.py index 0a21e0270398..41b659ac751b 100644 --- a/warehouse/oidc/__init__.py +++ b/warehouse/oidc/__init__.py @@ -10,8 +10,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from celery.schedules import crontab + from warehouse.oidc.interfaces import IOIDCPublisherService from warehouse.oidc.services import OIDCPublisherServiceFactory +from warehouse.oidc.tasks import compute_oidc_metrics from warehouse.oidc.utils import GITHUB_OIDC_ISSUER_URL, GOOGLE_OIDC_ISSUER_URL @@ -45,3 +48,6 @@ def includeme(config): config.add_route("oidc.audience", "/_/oidc/audience", domain=auth) config.add_route("oidc.github.mint_token", "/_/oidc/github/mint-token", domain=auth) + + # Compute OIDC metrics periodically + config.add_periodic_task(crontab(minute=0, hour=3), compute_oidc_metrics) diff --git a/warehouse/oidc/tasks.py b/warehouse/oidc/tasks.py new file mode 100644 index 000000000000..b37dd3cdd50b --- /dev/null +++ b/warehouse/oidc/tasks.py @@ -0,0 +1,24 @@ +from warehouse import tasks +from warehouse.metrics import IMetricsService +from warehouse.packaging.models import Project + + +@tasks.task(ignore_result=True, acks_late=True) +def compute_oidc_metrics(request): + metrics = request.find_service(IMetricsService, context=None) + + projects_using_oidc = ( + request.db.query(Project).distinct().join(Project.oidc_publishers) + ) + + # Metric for count of all projects using OIDC + metrics.gauge( + "warehouse.oidc.total_projects_using_oidc_publishers", + projects_using_oidc.count(), + ) + + # Metric for count of critical projects using OIDC + metrics.gauge( + "warehouse.oidc.total_critical_projects_using_oidc_publishers", + projects_using_oidc.where(Project.pypi_mandates_2fa.is_(True)).count(), + ) From 7c382bdf7b58f6f80388ab4a7ae8ccd911cd47ce Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Fri, 30 Jun 2023 10:22:46 -0500 Subject: [PATCH 2/3] Use metrics fixture, add license headers --- tests/unit/oidc/__init__.py | 11 +++++++++++ tests/unit/oidc/test_tasks.py | 19 ++++++++++++++----- warehouse/oidc/tasks.py | 12 ++++++++++++ 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/tests/unit/oidc/__init__.py b/tests/unit/oidc/__init__.py index e69de29bb2d1..164f68b09175 100644 --- a/tests/unit/oidc/__init__.py +++ b/tests/unit/oidc/__init__.py @@ -0,0 +1,11 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/unit/oidc/test_tasks.py b/tests/unit/oidc/test_tasks.py index f4427f562fed..05de93f8326d 100644 --- a/tests/unit/oidc/test_tasks.py +++ b/tests/unit/oidc/test_tasks.py @@ -1,3 +1,15 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import pretend from warehouse.oidc.tasks import compute_oidc_metrics @@ -6,7 +18,7 @@ from ...common.db.packaging import ProjectFactory -def test_compute_oidc_metrics(db_request, monkeypatch): +def test_compute_oidc_metrics(db_request, metrics): # Projects with OIDC critical_project_oidc = ProjectFactory.create( name="critical_project_oidc", pypi_mandates_2fa=True @@ -31,12 +43,9 @@ def test_compute_oidc_metrics(db_request, monkeypatch): ) oidc_publisher_2 = GitHubPublisherFactory.create(projects=[critical_project_oidc]) - gauge = pretend.call_recorder(lambda metric, value: None) - db_request.find_service = lambda *a, **kw: pretend.stub(gauge=gauge) - compute_oidc_metrics(db_request) - assert gauge.calls == [ + assert metrics.gauge.calls == [ pretend.call("warehouse.oidc.total_projects_using_oidc_publishers", 2), pretend.call("warehouse.oidc.total_critical_projects_using_oidc_publishers", 1), ] diff --git a/warehouse/oidc/tasks.py b/warehouse/oidc/tasks.py index b37dd3cdd50b..b623a2c9c7d8 100644 --- a/warehouse/oidc/tasks.py +++ b/warehouse/oidc/tasks.py @@ -1,3 +1,15 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from warehouse import tasks from warehouse.metrics import IMetricsService from warehouse.packaging.models import Project From b9d6c0c93b99ee4781e52d38377715818585c695 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Fri, 30 Jun 2023 11:35:29 -0500 Subject: [PATCH 3/3] Distinguish between OIDC being configured or successfully used --- tests/unit/oidc/test_tasks.py | 84 ++++++++++++++++++++++++++++++----- warehouse/oidc/tasks.py | 41 +++++++++++++---- 2 files changed, 104 insertions(+), 21 deletions(-) diff --git a/tests/unit/oidc/test_tasks.py b/tests/unit/oidc/test_tasks.py index 05de93f8326d..1ea53b923b8c 100644 --- a/tests/unit/oidc/test_tasks.py +++ b/tests/unit/oidc/test_tasks.py @@ -10,12 +10,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime + import pretend from warehouse.oidc.tasks import compute_oidc_metrics from ...common.db.oidc import GitHubPublisherFactory -from ...common.db.packaging import ProjectFactory +from ...common.db.packaging import ( + FileEventFactory, + FileFactory, + ProjectFactory, + ReleaseFactory, +) def test_compute_oidc_metrics(db_request, metrics): @@ -26,26 +33,79 @@ def test_compute_oidc_metrics(db_request, metrics): non_critical_project_oidc = ProjectFactory.create( name="non_critical_project_oidc", ) + non_released_critical_project_oidc = ProjectFactory.create( + name="non_released_critical_project_oidc", pypi_mandates_2fa=True + ) + non_released_project_oidc = ProjectFactory.create( + name="non_released_project_oidc", + ) # Projects without OIDC - critical_project_no_oidc = ProjectFactory.create( - name="critical_project_no_oidc", pypi_mandates_2fa=True - ) - non_critical_project_no_oidc = ProjectFactory.create( + ProjectFactory.create(name="critical_project_no_oidc", pypi_mandates_2fa=True) + ProjectFactory.create( name="non_critical_project_no_oidc", ) - # Create OIDC publishers. One OIDCPublisher can be shared by multiple - # projects so 'oidc_publisher_1' represents that situation. Verify - # that metrics don't double-count projects using multiple OIDC publishers. - oidc_publisher_1 = GitHubPublisherFactory.create( + # Create an OIDC publisher that's shared by multiple projects. + GitHubPublisherFactory.create( projects=[critical_project_oidc, non_critical_project_oidc] ) - oidc_publisher_2 = GitHubPublisherFactory.create(projects=[critical_project_oidc]) + + # Create an OIDC publisher that is only used by one project. + GitHubPublisherFactory.create(projects=[critical_project_oidc]) + + # Create OIDC publishers for projects which have no releases. + GitHubPublisherFactory.create(projects=[non_released_critical_project_oidc]) + GitHubPublisherFactory.create(projects=[non_released_project_oidc]) + + # Create some files which have/have not been published + # using OIDC in different scenarios. + + # Scenario: Same release, difference between files. + release_1 = ReleaseFactory.create(project=critical_project_oidc) + file_1_1 = FileFactory.create(release=release_1) + FileEventFactory.create( + source=file_1_1, + tag="fake:event", + time=datetime.datetime(2018, 2, 5, 17, 18, 18, 462_634), + additional={"publisher_url": "https://fake/url"}, + ) + + release_1 = ReleaseFactory.create(project=critical_project_oidc) + file_1_2 = FileFactory.create(release=release_1) + FileEventFactory.create( + source=file_1_2, + tag="fake:event", + time=datetime.datetime(2018, 2, 5, 17, 18, 18, 462_634), + ) + + # Scenario: Same project, differences between releases. + release_2 = ReleaseFactory.create(project=non_critical_project_oidc) + file_2 = FileFactory.create(release=release_2) + FileEventFactory.create( + source=file_2, + tag="fake:event", + time=datetime.datetime(2018, 2, 5, 17, 18, 18, 462_634), + additional={"publisher_url": "https://fake/url"}, + ) + + release_3 = ReleaseFactory.create(project=non_critical_project_oidc) + file_3 = FileFactory.create(release=release_3) + FileEventFactory.create( + source=file_3, + tag="fake:event", + time=datetime.datetime(2018, 2, 5, 17, 18, 18, 462_634), + ) compute_oidc_metrics(db_request) assert metrics.gauge.calls == [ - pretend.call("warehouse.oidc.total_projects_using_oidc_publishers", 2), - pretend.call("warehouse.oidc.total_critical_projects_using_oidc_publishers", 1), + pretend.call("warehouse.oidc.total_projects_configured_oidc_publishers", 4), + pretend.call( + "warehouse.oidc.total_critical_projects_configured_oidc_publishers", 2 + ), + pretend.call("warehouse.oidc.total_projects_published_with_oidc_publishers", 2), + pretend.call( + "warehouse.oidc.total_critical_projects_published_with_oidc_publishers", 1 + ), ] diff --git a/warehouse/oidc/tasks.py b/warehouse/oidc/tasks.py index b623a2c9c7d8..9265588c5edf 100644 --- a/warehouse/oidc/tasks.py +++ b/warehouse/oidc/tasks.py @@ -12,25 +12,48 @@ from warehouse import tasks from warehouse.metrics import IMetricsService -from warehouse.packaging.models import Project +from warehouse.packaging.models import File, Project, Release @tasks.task(ignore_result=True, acks_late=True) def compute_oidc_metrics(request): metrics = request.find_service(IMetricsService, context=None) - projects_using_oidc = ( - request.db.query(Project).distinct().join(Project.oidc_publishers) + projects_configured_oidc = ( + request.db.query(Project.id).distinct().join(Project.oidc_publishers) ) - # Metric for count of all projects using OIDC + # Metric for count of all projects that have configured OIDC. metrics.gauge( - "warehouse.oidc.total_projects_using_oidc_publishers", - projects_using_oidc.count(), + "warehouse.oidc.total_projects_configured_oidc_publishers", + projects_configured_oidc.count(), ) - # Metric for count of critical projects using OIDC + # Metric for count of critical projects that have configured OIDC. metrics.gauge( - "warehouse.oidc.total_critical_projects_using_oidc_publishers", - projects_using_oidc.where(Project.pypi_mandates_2fa.is_(True)).count(), + "warehouse.oidc.total_critical_projects_configured_oidc_publishers", + projects_configured_oidc.where(Project.pypi_mandates_2fa.is_(True)).count(), + ) + + # Need to check FileEvent.additional['publisher_url'] to determine which + # projects have successfully published via an OIDC publisher. + projects_published_with_oidc = ( + request.db.query(Project.id) + .distinct() + .join(Project.releases) + .join(Release.files) + .join(File.events) + .where(File.Event.additional.op("->>")("publisher_url").is_not(None)) + ) + + # Metric for count of all projects that have published via OIDC + metrics.gauge( + "warehouse.oidc.total_projects_published_with_oidc_publishers", + projects_published_with_oidc.count(), + ) + + # Metric for count of critical projects that have published via OIDC + metrics.gauge( + "warehouse.oidc.total_critical_projects_published_with_oidc_publishers", + projects_published_with_oidc.where(Project.pypi_mandates_2fa.is_(True)).count(), )