From cdf85048db52dea520e2380d6c6f6f88086138cd Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Wed, 24 Sep 2025 14:41:32 -0400 Subject: [PATCH 1/2] Revert "revert: remove organization survey email functionality (#18642)" This reverts commit 8545b13ec9febaf62e364ea0e920f5f8da429650. --- tests/unit/cli/test_organizations.py | 379 ++++++++++++++++++ tests/unit/email/test_init.py | 96 +++++ warehouse/cli/organizations.py | 175 ++++++++ warehouse/email/__init__.py | 19 + .../email/organization-survey/body.html | 22 + .../email/organization-survey/body.txt | 23 ++ .../email/organization-survey/subject.txt | 5 + 7 files changed, 719 insertions(+) create mode 100644 tests/unit/cli/test_organizations.py create mode 100644 warehouse/cli/organizations.py create mode 100644 warehouse/templates/email/organization-survey/body.html create mode 100644 warehouse/templates/email/organization-survey/body.txt create mode 100644 warehouse/templates/email/organization-survey/subject.txt diff --git a/tests/unit/cli/test_organizations.py b/tests/unit/cli/test_organizations.py new file mode 100644 index 000000000000..2f2f22ae2c2e --- /dev/null +++ b/tests/unit/cli/test_organizations.py @@ -0,0 +1,379 @@ +# SPDX-License-Identifier: Apache-2.0 + +import pretend +import pyramid.scripting +import transaction + +from warehouse.cli import organizations + + +class TestOrganizationsCLI: + def test_send_survey_emails_dry_run(self, monkeypatch, cli): + """Test dry run doesn't send emails""" + # Create test organizations + org1 = pretend.stub( + id="org1-id", + name="TestOrg1", + orgtype=pretend.stub(value="Community"), + is_active=True, + projects=[], # No projects + users=[ + pretend.stub( + username="user1", + email="user1@example.com", + primary_email=pretend.stub( + email="user1@example.com", verified=True + ), + ), + pretend.stub( + username="user2", + email="user2@example.com", + primary_email=pretend.stub( + email="user2@example.com", verified=True + ), + ), + ], + ) + + org2 = pretend.stub( + id="org2-id", + name="TestOrg2", + orgtype=pretend.stub(value="Company"), + is_active=True, + projects=[pretend.stub()], # Has projects + users=[ + pretend.stub( + username="user3", + email="user3@example.com", + primary_email=pretend.stub( + email="user3@example.com", verified=True + ), + ), + ], + ) + + # Mock the database query + query_result = pretend.stub( + filter=lambda *args: pretend.stub( + options=lambda *args: pretend.stub( + limit=lambda n: pretend.stub(all=lambda: [org1, org2]), + all=lambda: [org1, org2], + ) + ) + ) + + # Mock pyramid.scripting.prepare + mock_request = pretend.stub( + db=pretend.stub(query=lambda *args: query_result), + registry={"celery.app": pretend.stub()}, + ) + mock_env = { + "request": mock_request, + "closer": pretend.call_recorder(lambda: None), + } + prepare = pretend.call_recorder(lambda registry: mock_env) + monkeypatch.setattr(pyramid.scripting, "prepare", prepare) + + # No need to mock send_organization_survey_email for dry-run test + + # Create config + config = pretend.stub( + registry={ + "celery.app": pretend.stub(), + } + ) + + # Run the command with dry-run + result = cli.invoke( + organizations.send_survey_emails, + ["--dry-run", "--limit", "2"], + obj=config, + ) + + assert result.exit_code == 0 + assert "DRY RUN" in result.output + assert "Would send no_utilization_community survey to user1" in result.output + assert "Would send no_utilization_community survey to user2" in result.output + assert "Would send utilization_company survey to user3" in result.output + + def test_send_survey_emails_actual_send(self, monkeypatch, cli): + """Test actual email sending""" + # Create test organization + org = pretend.stub( + id="org-id", + name="TestOrg", + orgtype=pretend.stub(value="Community"), + is_active=True, + projects=[pretend.stub()], # Has projects + users=[ + pretend.stub( + username="user1", + email="user1@example.com", + primary_email=pretend.stub( + email="user1@example.com", verified=True + ), + ), + ], + ) + + # Mock the database query + query_result = pretend.stub( + filter=lambda *args: pretend.stub( + options=lambda *args: pretend.stub( + limit=lambda n: pretend.stub(all=lambda: [org]), + all=lambda: [org], + ) + ) + ) + + # Mock transaction manager + tm = pretend.stub( + begin=pretend.call_recorder(lambda: None), + commit=pretend.call_recorder(lambda: None), + ) + + # Mock pyramid.scripting.prepare + mock_request = pretend.stub( + db=pretend.stub(query=lambda *args: query_result), + registry={"celery.app": pretend.stub()}, + ) + mock_env = { + "request": mock_request, + "closer": pretend.call_recorder(lambda: None), + } + prepare = pretend.call_recorder(lambda registry: mock_env) + monkeypatch.setattr(pyramid.scripting, "prepare", prepare) + + # Mock transaction.TransactionManager + monkeypatch.setattr(transaction, "TransactionManager", lambda explicit: tm) + + # Mock _get_task + mock_get_task = pretend.call_recorder(lambda app, task_func: pretend.stub()) + monkeypatch.setattr("warehouse.tasks._get_task", mock_get_task) + + # Track email sends + send_email_calls = [] + + def mock_send_email(request, user, **kwargs): + send_email_calls.append((request, user, kwargs)) + + monkeypatch.setattr( + "warehouse.email.send_organization_survey_email", + mock_send_email, + ) + + # Create config + config = pretend.stub( + registry={ + "celery.app": pretend.stub(), + } + ) + + # Run the command + result = cli.invoke( + organizations.send_survey_emails, + ["--limit", "1"], + obj=config, + ) + + assert result.exit_code == 0 + assert "Successfully queued 1 emails" in result.output + assert "Queued utilization_community survey to user1" in result.output + assert len(send_email_calls) == 1 + assert tm.begin.calls == [pretend.call()] + assert tm.commit.calls == [pretend.call()] + + def test_send_survey_emails_categorization(self, monkeypatch, cli): + """Test correct categorization of organizations""" + # Create test organizations with all 4 categories + orgs = [ + # Utilization + Company + pretend.stub( + name="CompanyWithProjects", + orgtype=pretend.stub(value="Company"), + is_active=True, + projects=[pretend.stub()], + users=[ + pretend.stub( + username="user1", + email="user1@example.com", + primary_email=pretend.stub( + email="user1@example.com", verified=True + ), + ) + ], + ), + # Utilization + Community + pretend.stub( + name="CommunityWithProjects", + orgtype=pretend.stub(value="Community"), + is_active=True, + projects=[pretend.stub()], + users=[ + pretend.stub( + username="user2", + email="user2@example.com", + primary_email=pretend.stub( + email="user2@example.com", verified=True + ), + ) + ], + ), + # No Utilization + Company + pretend.stub( + name="CompanyNoProjects", + orgtype=pretend.stub(value="Company"), + is_active=True, + projects=[], + users=[ + pretend.stub( + username="user3", + email="user3@example.com", + primary_email=pretend.stub( + email="user3@example.com", verified=True + ), + ) + ], + ), + # No Utilization + Community + pretend.stub( + name="CommunityNoProjects", + orgtype=pretend.stub(value="Community"), + is_active=True, + projects=[], + users=[ + pretend.stub( + username="user4", + email="user4@example.com", + primary_email=pretend.stub( + email="user4@example.com", verified=True + ), + ) + ], + ), + ] + + # Mock the database query + query_result = pretend.stub( + filter=lambda *args: pretend.stub( + options=lambda *args: pretend.stub(all=lambda: orgs) + ) + ) + + # Mock pyramid.scripting.prepare + mock_request = pretend.stub( + db=pretend.stub(query=lambda *args: query_result), + registry={"celery.app": pretend.stub()}, + ) + mock_env = { + "request": mock_request, + "closer": pretend.call_recorder(lambda: None), + } + prepare = pretend.call_recorder(lambda registry: mock_env) + monkeypatch.setattr(pyramid.scripting, "prepare", prepare) + + # No need to mock send_organization_survey_email for dry-run test + + # Create config + config = pretend.stub( + registry={ + "celery.app": pretend.stub(), + } + ) + + # Run the command + result = cli.invoke( + organizations.send_survey_emails, + ["--dry-run"], + obj=config, + ) + + assert result.exit_code == 0 + assert "Utilization + Company: 1" in result.output + assert "Utilization + Community: 1" in result.output + assert "No Utilization + Company: 1" in result.output + assert "No Utilization + Community: 1" in result.output + assert "Total organizations processed: 4" in result.output + assert "Total emails to send: 4" in result.output + + def test_send_survey_emails_error_handling(self, monkeypatch, cli): + """Test error handling when sending emails fails""" + # Create test organization + org = pretend.stub( + name="TestOrg", + orgtype=pretend.stub(value="Community"), + is_active=True, + projects=[], + users=[ + pretend.stub( + username="user1", + email="user1@example.com", + primary_email=pretend.stub( + email="user1@example.com", verified=True + ), + ), + ], + ) + + # Mock the database query + query_result = pretend.stub( + filter=lambda *args: pretend.stub( + options=lambda *args: pretend.stub( + all=lambda: [org], + limit=lambda n: pretend.stub(all=lambda: [org]), + ) + ) + ) + + # Mock transaction manager + tm = pretend.stub( + begin=pretend.call_recorder(lambda: None), + commit=pretend.call_recorder(lambda: None), + ) + + # Mock pyramid.scripting.prepare + mock_request = pretend.stub( + db=pretend.stub(query=lambda *args: query_result), + registry={"celery.app": pretend.stub()}, + ) + mock_env = { + "request": mock_request, + "closer": pretend.call_recorder(lambda: None), + } + prepare = pretend.call_recorder(lambda registry: mock_env) + monkeypatch.setattr(pyramid.scripting, "prepare", prepare) + + # Mock transaction.TransactionManager + monkeypatch.setattr(transaction, "TransactionManager", lambda explicit: tm) + + # Mock _get_task + mock_get_task = pretend.call_recorder(lambda app, task_func: pretend.stub()) + monkeypatch.setattr("warehouse.tasks._get_task", mock_get_task) + + # Make send_email raise an exception + def mock_send_email(request, user, **kwargs): + raise Exception("Email sending failed") + + monkeypatch.setattr( + "warehouse.email.send_organization_survey_email", + mock_send_email, + ) + + # Create config + config = pretend.stub( + registry={ + "celery.app": pretend.stub(), + } + ) + + # Run the command + result = cli.invoke( + organizations.send_survey_emails, + [], + obj=config, + ) + + assert result.exit_code == 0 + assert "ERROR sending to user1: Email sending failed" in result.output + # Transaction should still be committed + assert tm.commit.calls == [pretend.call()] diff --git a/tests/unit/email/test_init.py b/tests/unit/email/test_init.py index 8c9fc824ade3..9043625dca45 100644 --- a/tests/unit/email/test_init.py +++ b/tests/unit/email/test_init.py @@ -6254,3 +6254,99 @@ def test_user_terms_of_service_updated( }, ) ] + + +class TestSendOrganizationSurveyEmail: + def test_send_organization_survey_email( + self, db_request, pyramid_config, monkeypatch + ): + user = UserFactory.create(with_verified_primary_email=True) + organization_name = "Test Organization" + survey_url = "https://example.com/survey/test" + organization_type = "Community" + has_projects = True + + subject_renderer = pyramid_config.testing_add_renderer( + "email/organization-survey/subject.txt" + ) + subject_renderer.string_response = "Email Subject" + body_renderer = pyramid_config.testing_add_renderer( + "email/organization-survey/body.txt" + ) + body_renderer.string_response = "Email Body" + html_renderer = pyramid_config.testing_add_renderer( + "email/organization-survey/body.html" + ) + html_renderer.string_response = "Email HTML Body" + + send_email = pretend.stub( + delay=pretend.call_recorder(lambda *args, **kwargs: None) + ) + db_request.task = pretend.call_recorder(lambda *args, **kwargs: send_email) + monkeypatch.setattr(email, "send_email", send_email) + + db_request.user = user + db_request.registry.settings = {"mail.sender": "noreply@example.com"} + + result = email.send_organization_survey_email( + db_request, + user, + organization_name=organization_name, + survey_url=survey_url, + organization_type=organization_type, + has_projects=has_projects, + ) + + assert result == { + "username": user.username, + "organization_name": organization_name, + "survey_url": survey_url, + "organization_type": organization_type, + "has_projects": has_projects, + } + subject_renderer.assert_( + username=user.username, + organization_name=organization_name, + survey_url=survey_url, + organization_type=organization_type, + has_projects=has_projects, + ) + body_renderer.assert_( + username=user.username, + organization_name=organization_name, + survey_url=survey_url, + organization_type=organization_type, + has_projects=has_projects, + ) + html_renderer.assert_( + username=user.username, + organization_name=organization_name, + survey_url=survey_url, + organization_type=organization_type, + has_projects=has_projects, + ) + assert db_request.task.calls == [pretend.call(send_email)] + assert send_email.delay.calls == [ + pretend.call( + f"{user.name} <{user.email}>", + { + "sender": None, + "subject": "Email Subject", + "body_text": "Email Body", + "body_html": ( + "\n\n" + "

Email HTML Body

\n\n" + ), + }, + { + "tag": "account:email:sent", + "user_id": user.id, + "additional": { + "from_": "noreply@example.com", + "to": user.email, + "subject": "Email Subject", + "redact_ip": False, + }, + }, + ) + ] diff --git a/warehouse/cli/organizations.py b/warehouse/cli/organizations.py new file mode 100644 index 000000000000..6b180f8cae78 --- /dev/null +++ b/warehouse/cli/organizations.py @@ -0,0 +1,175 @@ +# SPDX-License-Identifier: Apache-2.0 + +import click + +from warehouse.cli import warehouse + +# Survey URL constants +SURVEY_URLS = { + "utilization_company": "https://forms.gle/CTy9LXzxqNRBJqy87", + "utilization_community": "https://forms.gle/r4A7eXR3qSHLRJhE7", + "no_utilization_company": "https://forms.gle/zXVZhXwwKhHWrsPo9", + "no_utilization_community": "https://forms.gle/qG3nm1hFAmWDYRFcA", +} + + +@warehouse.group() +def organizations(): + """ + Manage operations for organizations. + """ + + +@organizations.command() +@click.option( + "--dry-run", + is_flag=True, + default=False, + help="Print what would be sent without actually sending emails", +) +@click.option( + "--limit", + type=int, + help="Limit the number of organizations to process (useful for testing)", +) +@click.pass_obj +def send_survey_emails(config, dry_run, limit): + """ + Send survey emails to all organization members based on organization type + and utilization status. + """ + # Import here to avoid circular imports + import functools + import time + + import pyramid.scripting + import transaction + + from sqlalchemy.orm import joinedload + + from warehouse.email import send_organization_survey_email + from warehouse.organizations.models import Organization + from warehouse.tasks import _get_task + + # Create a proper request using pyramid's scripting support + env = pyramid.scripting.prepare(registry=config.registry) + request = env["request"] + session = request.db + + # Set up transaction manager for proper Celery task queueing + request.tm = transaction.TransactionManager(explicit=True) + request.tm.begin() + + # Add the task method to the request so we can enqueue Celery tasks + celery_app = config.registry["celery.app"] + request.task = functools.partial(_get_task, celery_app) + + # Add timings attribute that metrics service expects + request.timings = {"new_request_start": time.time() * 1000} + + # Get all organizations + orgs_query = ( + session.query(Organization) + .filter(Organization.is_active.is_(True)) + .options( + joinedload(Organization.users), + joinedload(Organization.projects), + ) + ) + + if limit: + orgs_query = orgs_query.limit(limit) + + organizations = orgs_query.all() + + click.echo(f"Processing {len(organizations)} organizations...") + + stats = { + "total_orgs": 0, + "total_emails": 0, + "utilization_company": 0, + "utilization_community": 0, + "no_utilization_company": 0, + "no_utilization_community": 0, + } + + for org in organizations: + stats["total_orgs"] += 1 + + # Determine utilization status (has projects?) + has_projects = len(org.projects) > 0 + + # Determine organization type + is_company = org.orgtype.value == "Company" + + # Select appropriate survey URL + if has_projects and is_company: + survey_url = SURVEY_URLS["utilization_company"] + survey_type = "utilization_company" + elif has_projects and not is_company: + survey_url = SURVEY_URLS["utilization_community"] + survey_type = "utilization_community" + elif not has_projects and is_company: + survey_url = SURVEY_URLS["no_utilization_company"] + survey_type = "no_utilization_company" + else: + survey_url = SURVEY_URLS["no_utilization_community"] + survey_type = "no_utilization_community" + + stats[survey_type] += 1 + + # Get unique users for this organization + users = set(org.users) + + click.echo( + f" Organization: {org.name} " + f"(Type: {org.orgtype.value}, " + f"Has Projects: {has_projects}, " + f"Users: {len(users)})" + ) + + for user in users: + stats["total_emails"] += 1 + + if dry_run: + click.echo( + f" [DRY RUN] Would send {survey_type} survey to {user.username} " + f"({user.email}) for org {org.name}" + ) + else: + try: + send_organization_survey_email( + request, + user, + organization_name=org.name, + survey_url=survey_url, + organization_type=org.orgtype.value, + has_projects=has_projects, + ) + click.echo( + f" Queued {survey_type} survey to {user.username} " + f"({user.email})" + ) + except Exception as e: + click.echo( + f" ERROR sending to {user.username}: {str(e)}", err=True + ) + + # Print summary statistics + click.echo("\nSummary:") + click.echo(f" Total organizations processed: {stats['total_orgs']}") + click.echo(f" Total emails to send: {stats['total_emails']}") + click.echo(f" Utilization + Company: {stats['utilization_company']}") + click.echo(f" Utilization + Community: {stats['utilization_community']}") + click.echo(f" No Utilization + Company: {stats['no_utilization_company']}") + click.echo(f" No Utilization + Community: {stats['no_utilization_community']}") + + if dry_run: + click.echo("\nDRY RUN - No emails were actually sent") + else: + # Commit the transaction to actually enqueue the Celery tasks + request.tm.commit() + click.echo(f"\nSuccessfully queued {stats['total_emails']} emails") + + # Clean up the pyramid environment + env["closer"]() diff --git a/warehouse/email/__init__.py b/warehouse/email/__init__.py index 43dea91c09a5..e842029cd108 100644 --- a/warehouse/email/__init__.py +++ b/warehouse/email/__init__.py @@ -628,6 +628,25 @@ def send_organization_deleted_email(request, user, *, organization_name): } +@_email("organization-survey") +def send_organization_survey_email( + request, + user, + *, + organization_name, + survey_url, + organization_type, + has_projects, +): + return { + "username": user.username, + "organization_name": organization_name, + "survey_url": survey_url, + "organization_type": organization_type, + "has_projects": has_projects, + } + + @_email("team-created") def send_team_created_email(request, user, *, organization_name, team_name): return { diff --git a/warehouse/templates/email/organization-survey/body.html b/warehouse/templates/email/organization-survey/body.html new file mode 100644 index 000000000000..2b6d10720e44 --- /dev/null +++ b/warehouse/templates/email/organization-survey/body.html @@ -0,0 +1,22 @@ +{# SPDX-License-Identifier: Apache-2.0 -#} +{% extends "email/_base/body.html" %} +{% set site = request.registry.settings["site.name"] %} +{% block content %} +

Hello {{ username }},

+

Thank you for being a part of PyPI Organizations!

+

+ PyPI Organizations was initially created to provide more granular team and project management for large communities and organizations. We are looking to expand PyPI Organizations's features and would love to hear your feedback. +

+

+ Please fill out the survey below on how using PyPI Organizations is going for you and the {{ organization_name }} team + and what you would like to see PyPI Organizations offer in the near future. +

+

+ Take the Survey +

+

Feel free to be as detailed as possible in your responses and feedback.

+

– The PyPI Team

+{% endblock %} +{% block reason %} + You are receiving this because you are a member of the {{ organization_name }} organization on PyPI. If you are a member of multiple PyPI Organizations you may receive multiple emails. +{% endblock %} diff --git a/warehouse/templates/email/organization-survey/body.txt b/warehouse/templates/email/organization-survey/body.txt new file mode 100644 index 000000000000..20a836769122 --- /dev/null +++ b/warehouse/templates/email/organization-survey/body.txt @@ -0,0 +1,23 @@ +{# SPDX-License-Identifier: Apache-2.0 -#} +{% extends "email/_base/body.txt" %} + +{% block content %} +Hello {{ username }}, + +Thank you for being a part of PyPI Organizations! + +PyPI Organizations was initially created to provide more granular team and project management for large communities and organizations. We are looking to expand PyPI Organizations’s features and would love to hear your feedback. + +Please fill out the survey below on how using PyPI Organizations is going for you and the **{{ organization_name }}** team +and what you would like to see PyPI Organizations offer in the near future. + +Take the survey: {{ survey_url }} + +Feel free to be as detailed as possible in your responses and feedback. + +– The PyPI Team +{% endblock %} + +{% block reason %} +You are receiving this because you are a member of the **{{ organization_name }}** organization on PyPI. If you are a member of multiple PyPI Organizations you may receive multiple emails. +{% endblock %} diff --git a/warehouse/templates/email/organization-survey/subject.txt b/warehouse/templates/email/organization-survey/subject.txt new file mode 100644 index 000000000000..ad4a92dfb48d --- /dev/null +++ b/warehouse/templates/email/organization-survey/subject.txt @@ -0,0 +1,5 @@ +{# SPDX-License-Identifier: Apache-2.0 -#} + +{% extends "email/_base/subject.txt" %} + +{% block subject %}Help shape the future of PyPI Organizations for {{ organization_name }}{% endblock %} \ No newline at end of file From a0d1ee71d4c25f3077e5126ca868f59be2688044 Mon Sep 17 00:00:00 2001 From: Ee Durbin Date: Wed, 24 Sep 2025 14:44:52 -0400 Subject: [PATCH 2/2] update email template --- warehouse/templates/email/organization-survey/body.html | 7 +------ warehouse/templates/email/organization-survey/body.txt | 7 +------ warehouse/templates/email/organization-survey/subject.txt | 2 +- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/warehouse/templates/email/organization-survey/body.html b/warehouse/templates/email/organization-survey/body.html index 2b6d10720e44..86762335c5a3 100644 --- a/warehouse/templates/email/organization-survey/body.html +++ b/warehouse/templates/email/organization-survey/body.html @@ -3,13 +3,8 @@ {% set site = request.registry.settings["site.name"] %} {% block content %}

Hello {{ username }},

-

Thank you for being a part of PyPI Organizations!

- PyPI Organizations was initially created to provide more granular team and project management for large communities and organizations. We are looking to expand PyPI Organizations's features and would love to hear your feedback. -

-

- Please fill out the survey below on how using PyPI Organizations is going for you and the {{ organization_name }} team - and what you would like to see PyPI Organizations offer in the near future. + We recently sent out a survey to hear from you about PyPI Orgs. If you have already filled out the survey, thank you. For those of you who have not yet filled out the survey, please take some time to do so. We deeply value your feedback.

Take the Survey diff --git a/warehouse/templates/email/organization-survey/body.txt b/warehouse/templates/email/organization-survey/body.txt index 20a836769122..4bdb2389d496 100644 --- a/warehouse/templates/email/organization-survey/body.txt +++ b/warehouse/templates/email/organization-survey/body.txt @@ -4,12 +4,7 @@ {% block content %} Hello {{ username }}, -Thank you for being a part of PyPI Organizations! - -PyPI Organizations was initially created to provide more granular team and project management for large communities and organizations. We are looking to expand PyPI Organizations’s features and would love to hear your feedback. - -Please fill out the survey below on how using PyPI Organizations is going for you and the **{{ organization_name }}** team -and what you would like to see PyPI Organizations offer in the near future. +We recently sent out a survey to hear from you about PyPI Orgs. If you have already filled out the survey, thank you. For those of you who have not yet filled out the survey, please take some time to do so. We deeply value your feedback. Take the survey: {{ survey_url }} diff --git a/warehouse/templates/email/organization-survey/subject.txt b/warehouse/templates/email/organization-survey/subject.txt index ad4a92dfb48d..5bf7ca06836b 100644 --- a/warehouse/templates/email/organization-survey/subject.txt +++ b/warehouse/templates/email/organization-survey/subject.txt @@ -2,4 +2,4 @@ {% extends "email/_base/subject.txt" %} -{% block subject %}Help shape the future of PyPI Organizations for {{ organization_name }}{% endblock %} \ No newline at end of file +{% block subject %}Reminder: Help shape the future of PyPI Organizations for {{ organization_name }}{% endblock %}