Add metrics for projects using OIDC publishers

tests/unit/oidc/__init__.py

@@ -0,0 +1,11 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/unit/oidc/test_tasks.py

@@ -0,0 +1,111 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+
+import pretend
+
+from warehouse.oidc.tasks import compute_oidc_metrics
+
+from ...common.db.oidc import GitHubPublisherFactory
+from ...common.db.packaging import (
+    FileEventFactory,
+    FileFactory,
+    ProjectFactory,
+    ReleaseFactory,
+)
+
+
+def test_compute_oidc_metrics(db_request, metrics):
+    # Projects with OIDC
+    critical_project_oidc = ProjectFactory.create(
+        name="critical_project_oidc", pypi_mandates_2fa=True
+    )
+    non_critical_project_oidc = ProjectFactory.create(
+        name="non_critical_project_oidc",
+    )
+    non_released_critical_project_oidc = ProjectFactory.create(
+        name="non_released_critical_project_oidc", pypi_mandates_2fa=True
+    )
+    non_released_project_oidc = ProjectFactory.create(
+        name="non_released_project_oidc",
+    )
+
+    # Projects without OIDC
+    ProjectFactory.create(name="critical_project_no_oidc", pypi_mandates_2fa=True)
+    ProjectFactory.create(
+        name="non_critical_project_no_oidc",
+    )
+
+    # Create an OIDC publisher that's shared by multiple projects.
+    GitHubPublisherFactory.create(
+        projects=[critical_project_oidc, non_critical_project_oidc]
+    )
+
+    # Create an OIDC publisher that is only used by one project.
+    GitHubPublisherFactory.create(projects=[critical_project_oidc])
+
+    # Create OIDC publishers for projects which have no releases.
+    GitHubPublisherFactory.create(projects=[non_released_critical_project_oidc])
+    GitHubPublisherFactory.create(projects=[non_released_project_oidc])
+
+    # Create some files which have/have not been published
+    # using OIDC in different scenarios.
+
+    # Scenario: Same release, difference between files.
+    release_1 = ReleaseFactory.create(project=critical_project_oidc)
+    file_1_1 = FileFactory.create(release=release_1)
+    FileEventFactory.create(
+        source=file_1_1,
+        tag="fake:event",
+        time=datetime.datetime(2018, 2, 5, 17, 18, 18, 462_634),
+        additional={"publisher_url": "https://fake/url"},
+    )
+
+    release_1 = ReleaseFactory.create(project=critical_project_oidc)
+    file_1_2 = FileFactory.create(release=release_1)
+    FileEventFactory.create(
+        source=file_1_2,
+        tag="fake:event",
+        time=datetime.datetime(2018, 2, 5, 17, 18, 18, 462_634),
+    )
+
+    # Scenario: Same project, differences between releases.
+    release_2 = ReleaseFactory.create(project=non_critical_project_oidc)
+    file_2 = FileFactory.create(release=release_2)
+    FileEventFactory.create(
+        source=file_2,
+        tag="fake:event",
+        time=datetime.datetime(2018, 2, 5, 17, 18, 18, 462_634),
+        additional={"publisher_url": "https://fake/url"},
+    )
+
+    release_3 = ReleaseFactory.create(project=non_critical_project_oidc)
+    file_3 = FileFactory.create(release=release_3)
+    FileEventFactory.create(
+        source=file_3,
+        tag="fake:event",
+        time=datetime.datetime(2018, 2, 5, 17, 18, 18, 462_634),
+    )
+
+    compute_oidc_metrics(db_request)
+
+    assert metrics.gauge.calls == [
+        pretend.call("warehouse.oidc.total_projects_configured_oidc_publishers", 4),
+        pretend.call(
+            "warehouse.oidc.total_critical_projects_configured_oidc_publishers", 2
+        ),
+        pretend.call("warehouse.oidc.total_projects_published_with_oidc_publishers", 2),
+        pretend.call(
+            "warehouse.oidc.total_critical_projects_published_with_oidc_publishers", 1
+        ),
+    ]

warehouse/oidc/__init__.py

@@ -10,8 +10,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from celery.schedules import crontab
+
 from warehouse.oidc.interfaces import IOIDCPublisherService
 from warehouse.oidc.services import OIDCPublisherServiceFactory
+from warehouse.oidc.tasks import compute_oidc_metrics
 from warehouse.oidc.utils import GITHUB_OIDC_ISSUER_URL, GOOGLE_OIDC_ISSUER_URL
 
 
@@ -45,3 +48,6 @@ def includeme(config):
 
     config.add_route("oidc.audience", "/_/oidc/audience", domain=auth)
     config.add_route("oidc.github.mint_token", "/_/oidc/github/mint-token", domain=auth)
+
+    # Compute OIDC metrics periodically
+    config.add_periodic_task(crontab(minute=0, hour=3), compute_oidc_metrics)

warehouse/oidc/tasks.py

@@ -0,0 +1,59 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from warehouse import tasks
+from warehouse.metrics import IMetricsService
+from warehouse.packaging.models import File, Project, Release
+
+
+@tasks.task(ignore_result=True, acks_late=True)
+def compute_oidc_metrics(request):
+    metrics = request.find_service(IMetricsService, context=None)
+
+    projects_configured_oidc = (
+        request.db.query(Project.id).distinct().join(Project.oidc_publishers)
+    )
+
+    # Metric for count of all projects that have configured OIDC.
+    metrics.gauge(
+        "warehouse.oidc.total_projects_configured_oidc_publishers",
+        projects_configured_oidc.count(),
+    )
+
+    # Metric for count of critical projects that have configured OIDC.
+    metrics.gauge(
+        "warehouse.oidc.total_critical_projects_configured_oidc_publishers",
+        projects_configured_oidc.where(Project.pypi_mandates_2fa.is_(True)).count(),
+    )
+
+    # Need to check FileEvent.additional['publisher_url'] to determine which
+    # projects have successfully published via an OIDC publisher.
+    projects_published_with_oidc = (
+        request.db.query(Project.id)
+        .distinct()
+        .join(Project.releases)
+        .join(Release.files)
+        .join(File.events)
+        .where(File.Event.additional.op("->>")("publisher_url").is_not(None))
+    )
+
+    # Metric for count of all projects that have published via OIDC
+    metrics.gauge(
+        "warehouse.oidc.total_projects_published_with_oidc_publishers",
+        projects_published_with_oidc.count(),
+    )
+
+    # Metric for count of critical projects that have published via OIDC
+    metrics.gauge(
+        "warehouse.oidc.total_critical_projects_published_with_oidc_publishers",
+        projects_published_with_oidc.where(Project.pypi_mandates_2fa.is_(True)).count(),
+    )