From ded4fab2075382d6447aefcfe46c0fde9ba88e40 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 25 May 2023 13:40:13 +0200 Subject: [PATCH 01/31] Config: deprecated notification for builds without config file When we detect a build is built without a Read the Docs configuration file (`.readthedocs.yaml`) we show multiple notifications: - a static warning message in the build detail's page - a persistent on-site notification to all maintainers/admin of the project - send a weekly email (at most) This is the initial step to attempt making users to migrate to our config file v2, giving them a enough window to do this and avoid breaking their builds in the future. Closes #10348 --- readthedocs/builds/models.py | 18 ++++- readthedocs/projects/tasks/builds.py | 13 ++- readthedocs/projects/tasks/utils.py | 81 +++++++++++++++++++ readthedocs/rtd_tests/tests/test_builds.py | 6 +- .../templates/builds/build_detail.html | 12 +-- .../deprecated_config_file_used_email.html | 1 + .../deprecated_config_file_used_email.txt | 9 +++ 7 files changed, 127 insertions(+), 13 deletions(-) create mode 100644 readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html create mode 100644 readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index a2cc5bc1348..abdc2f28041 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -1069,10 +1069,20 @@ def can_rebuild(self): def external_version_name(self): return external_version_name(self) - def using_latest_config(self): - if self.config: - return int(self.config.get('version', '1')) == LATEST_CONFIGURATION_VERSION - return False + def deprecated_config_used(self): + """ + Check whether this particular build is using a deprecated config file. + + When using v1 or not having a config file at all, it returns ``True``. + Returns ``False`` only when it has a config file and it is using v2. + + Note we are using this to communicate deprecation of v1 file and not using a config file. + See https://github.com/readthedocs/readthedocs.org/issues/10342 + """ + if not self.config: + return True + + return int(self.config.get("version", "1")) != LATEST_CONFIGURATION_VERSION def reset(self): """ diff --git a/readthedocs/projects/tasks/builds.py b/readthedocs/projects/tasks/builds.py index 55b00618351..8a47b397e72 100644 --- a/readthedocs/projects/tasks/builds.py +++ b/readthedocs/projects/tasks/builds.py @@ -66,7 +66,12 @@ from ..signals import before_vcs from .mixins import SyncRepositoryMixin from .search import fileify -from .utils import BuildRequest, clean_build, send_external_build_status +from .utils import ( + BuildRequest, + clean_build, + deprecated_config_file_used_notification, + send_external_build_status, +) log = structlog.get_logger(__name__) @@ -679,6 +684,12 @@ def after_return(self, status, retval, task_id, args, kwargs, einfo): if self.data.build.get("state") not in BUILD_FINAL_STATES: build_state = BUILD_STATE_FINISHED + # Trigger a Celery task here to check if the build is using v1 or not a + # config file at all to create a on-site/email notifications. Note we + # can't create the notification from here since we don't have access to + # the database from the builders. + deprecated_config_file_used_notification.delay(self.data.build["id"]) + self.update_build(build_state) self.save_build_data() diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index da5a16c667f..f9eee25a3ad 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -3,9 +3,11 @@ import structlog from celery.worker.request import Request +from django.core.cache import cache from django.db.models import Q from django.utils import timezone from django.utils.translation import gettext_lazy as _ +from messages_extends.constants import WARNING_PERSISTENT from readthedocs.builds.constants import ( BUILD_FINAL_STATES, @@ -14,7 +16,11 @@ ) from readthedocs.builds.models import Build from readthedocs.builds.tasks import send_build_status +from readthedocs.core.permissions import AdminPermission from readthedocs.core.utils.filesystem import safe_rmtree +from readthedocs.notifications import Notification, SiteNotification +from readthedocs.notifications.backends import EmailBackend +from readthedocs.notifications.constants import REQUIREMENT from readthedocs.storage import build_media_storage from readthedocs.worker import app @@ -154,6 +160,81 @@ def send_external_build_status(version_type, build_pk, commit, status): send_build_status.delay(build_pk, commit, status) +class DeprecatedConfigFileSiteNotification(SiteNotification): + + failure_message = ( + "Your project '{{ object.slug }}' doesn't have a " + '.readthedocs.yaml ' + "configuration file. " + "This feature is deprecated and will be removed soon. " + "Make sure to create one for your project to keep your builds working." + ) + failure_level = WARNING_PERSISTENT + + +class DeprecatedConfigFileEmailNotification(Notification): + + app_templates = "projects" + name = "deprecated_config_file_used" + context_object_name = "project" + subject = "Your project will start failing soon" + level = REQUIREMENT + + def send(self): + """Method overwritten to remove on-site backend.""" + backend = EmailBackend(self.request) + backend.send(self) + + +@app.task(queue="web") +def deprecated_config_file_used_notification(build_pk): + """ + Create a notification about not using a config file for all the maintainers of the project. + + This task is triggered by the build process to be executed on the webs, + since we don't have access to the db from the build. + """ + build = Build.objects.filter(pk=build_pk).first() + if not build or not build.deprecated_config_used: + return + + log.bind( + build_pk=build_pk, + project_slug=build.project.slug, + ) + + users = AdminPermission.owners(build.project) + log.bind(users=len(users)) + + log.info("Sending deprecation config file onsite notification.") + for user in users: + n = DeprecatedConfigFileSiteNotification( + user=user, + context_object=build.project, + success=False, + ) + n.send() + + # Send email notifications only once a week + cache_prefix = "deprecated-config-file-notification" + cached = cache.get(f"{cache_prefix}-{build.project.slug}") + if cached: + log.info("Deprecation config file email sent recently. Skipping.") + return + + log.info("Sending deprecation config file email notification.") + for user in users: + n = DeprecatedConfigFileEmailNotification( + user=user, + context_object=build.project, + ) + n.send() + + # Cache this notification for a week + # TODO: reduce this notification period to 3 days after having this deployed for some weeks + cache.set(f"{cache_prefix}-{build.project.slug}", "sent", timeout=7 * 24 * 60 * 60) + + class BuildRequest(Request): def on_timeout(self, soft, timeout): diff --git a/readthedocs/rtd_tests/tests/test_builds.py b/readthedocs/rtd_tests/tests/test_builds.py index 2a4da4e1092..583f48a5307 100644 --- a/readthedocs/rtd_tests/tests/test_builds.py +++ b/readthedocs/rtd_tests/tests/test_builds.py @@ -249,7 +249,7 @@ def test_build_is_stale(self): self.assertTrue(build_two.is_stale) self.assertFalse(build_three.is_stale) - def test_using_latest_config(self): + def test_deprecated_config_used(self): now = timezone.now() build = get( @@ -260,12 +260,12 @@ def test_using_latest_config(self): state='finished', ) - self.assertFalse(build.using_latest_config()) + self.assertFalse(build.deprecated_config_used()) build.config = {'version': 2} build.save() - self.assertTrue(build.using_latest_config()) + self.assertTrue(build.deprecated_config_used()) def test_build_is_external(self): # Turn the build version to EXTERNAL type. diff --git a/readthedocs/templates/builds/build_detail.html b/readthedocs/templates/builds/build_detail.html index e47d129ae9a..e23ff920bb5 100644 --- a/readthedocs/templates/builds/build_detail.html +++ b/readthedocs/templates/builds/build_detail.html @@ -161,18 +161,20 @@

{% endif %} - {% if build.finished and not build.using_latest_config %} + + {# This message is not dynamic and only appears when loading the page after the build has finished #} + {% if build.finished and build.deprecated_config_used %}

{% blocktrans trimmed with config_file_link="https://docs.readthedocs.io/page/config-file/v2.html" %} - Configure your documentation builds! - Adding a .readthedocs.yaml file to your project - is the recommended way to configure your documentation builds. - You can declare dependencies, set up submodules, and many other great features. + Your builds will stop working soon!
+ Building without a configuration file (or using v1) is deprecated and will be removed soon. + Add a .readthedocs.yaml config file to your project to keep your builds working. {% endblocktrans %}

{% endif %} + {% endif %} {% if build.finished and build.config.build.commands %} diff --git a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html new file mode 100644 index 00000000000..5f4646f18d1 --- /dev/null +++ b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html @@ -0,0 +1 @@ + diff --git a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt new file mode 100644 index 00000000000..095ff38f5b2 --- /dev/null +++ b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt @@ -0,0 +1,9 @@ +{% extends "core/email/common.txt" %} +{% block content %} +Your project "{{ project.slug }}" is not using a .readthedocs.yaml configuration file and will stop working soon. + +We strongly recommend you to add a configuration file to keep your builds working. + +Get in touch with us at {{ production_uri }}{% url 'support' %} +and let us know if you are unable to migrate to a config file for any reason. +{% endblock %} From 931ba6201eca8403cdb3e65b68d7cccee6ea4078 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 25 May 2023 15:21:14 +0200 Subject: [PATCH 02/31] Test: invert logic --- readthedocs/rtd_tests/tests/test_builds.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_builds.py b/readthedocs/rtd_tests/tests/test_builds.py index 583f48a5307..a58c47385f4 100644 --- a/readthedocs/rtd_tests/tests/test_builds.py +++ b/readthedocs/rtd_tests/tests/test_builds.py @@ -260,12 +260,12 @@ def test_deprecated_config_used(self): state='finished', ) - self.assertFalse(build.deprecated_config_used()) + self.assertTrue(build.deprecated_config_used()) build.config = {'version': 2} build.save() - self.assertTrue(build.deprecated_config_used()) + self.assertFalse(build.deprecated_config_used()) def test_build_is_external(self): # Turn the build version to EXTERNAL type. From a623ac6bdcd388414f564d966ffd1e1853a490e1 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 May 2023 10:18:29 +0200 Subject: [PATCH 03/31] Notification's copy: feedback from review --- readthedocs/projects/tasks/utils.py | 6 ++++-- readthedocs/templates/builds/build_detail.html | 4 ++-- .../deprecated_config_file_used_email.html | 11 ++++++++++- .../deprecated_config_file_used_email.txt | 7 ++++--- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index f9eee25a3ad..d382a463655 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -166,8 +166,10 @@ class DeprecatedConfigFileSiteNotification(SiteNotification): "Your project '{{ object.slug }}' doesn't have a " '.readthedocs.yaml ' "configuration file. " - "This feature is deprecated and will be removed soon. " - "Make sure to create one for your project to keep your builds working." + "Configuration files will soon be required by projects, " + "and will no longer be optional. " + "Make sure to create one for your project to ensure your project continues " + "building successfully." ) failure_level = WARNING_PERSISTENT diff --git a/readthedocs/templates/builds/build_detail.html b/readthedocs/templates/builds/build_detail.html index e23ff920bb5..4f12852835a 100644 --- a/readthedocs/templates/builds/build_detail.html +++ b/readthedocs/templates/builds/build_detail.html @@ -168,8 +168,8 @@

{% blocktrans trimmed with config_file_link="https://docs.readthedocs.io/page/config-file/v2.html" %} Your builds will stop working soon!
- Building without a configuration file (or using v1) is deprecated and will be removed soon. - Add a .readthedocs.yaml config file to your project to keep your builds working. + Configuration files will soon be required by projects, and will no longer be optional. + Add a .readthedocs.yaml config file to your project to ensure your project continues building successfully. {% endblocktrans %}

diff --git a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html index 5f4646f18d1..03eb3c9e642 100644 --- a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html +++ b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html @@ -1 +1,10 @@ - +{% extends "core/email/common.html" %} +{% block content %} +Your project "{{ project.slug }}" is not using a .readthedocs.yaml configuration file and will stop working soon. + +Configuration files will soon be required by projects, and will no longer be optional. +You will need to add a configuration file to your project to ensure your project continues building successfully. + +Get in touch with us via our support +and let us know if you are unable to use a configuration file for any reason. +{% endblock %} diff --git a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt index 095ff38f5b2..78ac179bbb8 100644 --- a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt +++ b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt @@ -1,9 +1,10 @@ {% extends "core/email/common.txt" %} {% block content %} -Your project "{{ project.slug }}" is not using a .readthedocs.yaml configuration file and will stop working soon. +Your project "{{ project.slug }}" is not using a .readthedocs.yaml configuration file (https://docs.readthedocs.io/en/stable/config-file/v2.html) and will stop working soon. -We strongly recommend you to add a configuration file to keep your builds working. +Configuration files will soon be required by projects, and will no longer be optional. +You will need to add a configuration file to your project to ensure your project continues building successfully. Get in touch with us at {{ production_uri }}{% url 'support' %} -and let us know if you are unable to migrate to a config file for any reason. +and let us know if you are unable to use a configuration file for any reason. {% endblock %} From 1c688b4b61375cbbbdaeeb441ef0842ebbdd6b18 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 May 2023 11:21:34 +0200 Subject: [PATCH 04/31] It's a function --- readthedocs/projects/tasks/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index d382a463655..daf735d36c1 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -197,7 +197,7 @@ def deprecated_config_file_used_notification(build_pk): since we don't have access to the db from the build. """ build = Build.objects.filter(pk=build_pk).first() - if not build or not build.deprecated_config_used: + if not build or not build.deprecated_config_used(): return log.bind( From 22eee3c3ea27a34fb0eb6b20f264468155021806 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 May 2023 12:06:10 +0200 Subject: [PATCH 05/31] Notifications: use a scheduled Celery task to send them Instead of sending an onsite notification on each build, we use a scheduled Celery task that runs once a week to send them. It filter projects that are not using the v2 config file. --- readthedocs/projects/tasks/builds.py | 13 +---- readthedocs/projects/tasks/utils.py | 82 +++++++++++++++------------- readthedocs/settings/base.py | 5 ++ 3 files changed, 49 insertions(+), 51 deletions(-) diff --git a/readthedocs/projects/tasks/builds.py b/readthedocs/projects/tasks/builds.py index 8a47b397e72..55b00618351 100644 --- a/readthedocs/projects/tasks/builds.py +++ b/readthedocs/projects/tasks/builds.py @@ -66,12 +66,7 @@ from ..signals import before_vcs from .mixins import SyncRepositoryMixin from .search import fileify -from .utils import ( - BuildRequest, - clean_build, - deprecated_config_file_used_notification, - send_external_build_status, -) +from .utils import BuildRequest, clean_build, send_external_build_status log = structlog.get_logger(__name__) @@ -684,12 +679,6 @@ def after_return(self, status, retval, task_id, args, kwargs, einfo): if self.data.build.get("state") not in BUILD_FINAL_STATES: build_state = BUILD_STATE_FINISHED - # Trigger a Celery task here to check if the build is using v1 or not a - # config file at all to create a on-site/email notifications. Note we - # can't create the notification from here since we don't have access to - # the database from the builders. - deprecated_config_file_used_notification.delay(self.data.build["id"]) - self.update_build(build_state) self.save_build_data() diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index daf735d36c1..9b171496b41 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -3,7 +3,6 @@ import structlog from celery.worker.request import Request -from django.core.cache import cache from django.db.models import Q from django.utils import timezone from django.utils.translation import gettext_lazy as _ @@ -21,6 +20,7 @@ from readthedocs.notifications import Notification, SiteNotification from readthedocs.notifications.backends import EmailBackend from readthedocs.notifications.constants import REQUIREMENT +from readthedocs.projects.models import Project from readthedocs.storage import build_media_storage from readthedocs.worker import app @@ -189,52 +189,56 @@ def send(self): @app.task(queue="web") -def deprecated_config_file_used_notification(build_pk): +def deprecated_config_file_used_notification(): """ Create a notification about not using a config file for all the maintainers of the project. - This task is triggered by the build process to be executed on the webs, - since we don't have access to the db from the build. + This is a scheduled task to be executed on the webs. + Note the code uses `.iterator` and `.only` to avoid killing the db with this query. """ - build = Build.objects.filter(pk=build_pk).first() - if not build or not build.deprecated_config_used(): - return + projects = set() + # NOTE: we could skip the projects with a spam score > 150, + # to reduce the amount of email/onsite notifications we send + start_datetime = datetime.datetime.now() + for project in Project.objects.all().only("slug", "default_version").iterator(): + # NOTE: instead of iterating over all the active versions, + # we can only consider the default one + for version in ( + project.versions.filter(Q(active=True) | Q(slug=project.default_version)) + # Add a limit of 15 to protect ourselves against projects with + # hundred of active versions + .only("id")[:15].iterator() + ): + build = version.builds.filter(success=True).only("_config").first() + if build and build.deprecated_config_used(): + projects.add(project.slug) + # Do not continue iterating over the + # other versions when we know this project + # will get the notification + break - log.bind( - build_pk=build_pk, - project_slug=build.project.slug, + log.info( + "Sending deprecated config file notification.", + query_seconds=(datetime.datetime.now() - start_datetime).seconds, + projects=len(projects), ) - users = AdminPermission.owners(build.project) - log.bind(users=len(users)) - - log.info("Sending deprecation config file onsite notification.") - for user in users: - n = DeprecatedConfigFileSiteNotification( - user=user, - context_object=build.project, - success=False, - ) - n.send() - - # Send email notifications only once a week - cache_prefix = "deprecated-config-file-notification" - cached = cache.get(f"{cache_prefix}-{build.project.slug}") - if cached: - log.info("Deprecation config file email sent recently. Skipping.") - return - - log.info("Sending deprecation config file email notification.") - for user in users: - n = DeprecatedConfigFileEmailNotification( - user=user, - context_object=build.project, - ) - n.send() + for project in Project.objects.filter(slug__in=projects): + users = AdminPermission.owners(project) + for user in users: + n = DeprecatedConfigFileSiteNotification( + user=user, + context_object=project, + success=False, + ) + n.send() - # Cache this notification for a week - # TODO: reduce this notification period to 3 days after having this deployed for some weeks - cache.set(f"{cache_prefix}-{build.project.slug}", "sent", timeout=7 * 24 * 60 * 60) + for user in users: + n = DeprecatedConfigFileEmailNotification( + user=user, + context_object=project, + ) + n.send() class BuildRequest(Request): diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 415e261cb5a..8bad09e853e 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -532,6 +532,11 @@ def TEMPLATES(self): 'schedule': crontab(minute='*/15'), 'options': {'queue': 'web'}, }, + 'weekly-config-file-notification': { + 'task': 'readthedocs.projects.tasks.utils.deprecated_config_file_used_notification', + 'schedule': crontab(day_of_week='wednesday', hour=11, minute=15), + 'options': {'queue': 'web'}, + }, } MULTIPLE_BUILD_SERVERS = [CELERY_DEFAULT_QUEUE] From 035c81f6b56650d26c13c49a044f2352ca7021b9 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 31 May 2023 11:00:19 +0200 Subject: [PATCH 06/31] Feedback from the review --- readthedocs/projects/tasks/utils.py | 29 ++++++++++++------- .../deprecated_config_file_used_email.html | 2 +- .../deprecated_config_file_used_email.txt | 5 +++- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 9b171496b41..d870061ef65 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -164,8 +164,8 @@ class DeprecatedConfigFileSiteNotification(SiteNotification): failure_message = ( "Your project '{{ object.slug }}' doesn't have a " - '.readthedocs.yaml ' - "configuration file. " + '.readthedocs.yaml ' + "configuration file. " "Configuration files will soon be required by projects, " "and will no longer be optional. " "Make sure to create one for your project to ensure your project continues " @@ -179,7 +179,7 @@ class DeprecatedConfigFileEmailNotification(Notification): app_templates = "projects" name = "deprecated_config_file_used" context_object_name = "project" - subject = "Your project will start failing soon" + subject = "Add a configuration file to your project to continue building" level = REQUIREMENT def send(self): @@ -205,6 +205,7 @@ def deprecated_config_file_used_notification(): # we can only consider the default one for version in ( project.versions.filter(Q(active=True) | Q(slug=project.default_version)) + .order_by("-modified") # Add a limit of 15 to protect ourselves against projects with # hundred of active versions .only("id")[:15].iterator() @@ -217,28 +218,36 @@ def deprecated_config_file_used_notification(): # will get the notification break + n_projects = len(projects) log.info( "Sending deprecated config file notification.", query_seconds=(datetime.datetime.now() - start_datetime).seconds, - projects=len(projects), + projects=n_projects, ) - for project in Project.objects.filter(slug__in=projects): + projects = Project.objects.filter(slug__in=projects) + for i, project in enumerate(projects): + + if i % 500 == 0: + log.info( + "Sending deprecated config file notifications.", + progress=f"{i}/{n_projects}", + ) + users = AdminPermission.owners(project) for user in users: - n = DeprecatedConfigFileSiteNotification( + n_site = DeprecatedConfigFileSiteNotification( user=user, context_object=project, success=False, ) - n.send() + n_site.send() - for user in users: - n = DeprecatedConfigFileEmailNotification( + n_email = DeprecatedConfigFileEmailNotification( user=user, context_object=project, ) - n.send() + n_email.send() class BuildRequest(Request): diff --git a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html index 03eb3c9e642..2092399b5fc 100644 --- a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html +++ b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html @@ -1,6 +1,6 @@ {% extends "core/email/common.html" %} {% block content %} -Your project "{{ project.slug }}" is not using a .readthedocs.yaml configuration file and will stop working soon. +Your project "{{ project.slug }}" is not using a .readthedocs.yaml configuration file and builds will stop working soon. Configuration files will soon be required by projects, and will no longer be optional. You will need to add a configuration file to your project to ensure your project continues building successfully. diff --git a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt index 78ac179bbb8..b93ed737c7d 100644 --- a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt +++ b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt @@ -1,10 +1,13 @@ {% extends "core/email/common.txt" %} {% block content %} -Your project "{{ project.slug }}" is not using a .readthedocs.yaml configuration file (https://docs.readthedocs.io/en/stable/config-file/v2.html) and will stop working soon. +Your project "{{ project.slug }}" is not using a .readthedocs.yaml configuration file and builds will stop working soon. Configuration files will soon be required by projects, and will no longer be optional. You will need to add a configuration file to your project to ensure your project continues building successfully. +For more information on our configuration file, see: +https://docs.readthedocs.io/en/stable/config-file/v2.html + Get in touch with us at {{ production_uri }}{% url 'support' %} and let us know if you are unable to use a configuration file for any reason. {% endblock %} From 0a4156f31898823be7c98216f025c68fd6d5631a Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 31 May 2023 11:13:14 +0200 Subject: [PATCH 07/31] Darker failed on CircleCI because of this --- readthedocs/projects/tasks/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index d870061ef65..47c85a7591f 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -208,7 +208,8 @@ def deprecated_config_file_used_notification(): .order_by("-modified") # Add a limit of 15 to protect ourselves against projects with # hundred of active versions - .only("id")[:15].iterator() + .only("id")[:15] + .iterator() ): build = version.builds.filter(success=True).only("_config").first() if build and build.deprecated_config_used(): From 4044819d5533c35e91e4f16f8e3e89735f594bf1 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 31 May 2023 11:41:05 +0200 Subject: [PATCH 08/31] Links pointing to blog post --- readthedocs/projects/tasks/utils.py | 7 +++---- readthedocs/templates/builds/build_detail.html | 5 +++-- .../notifications/deprecated_config_file_used_email.html | 5 ++++- .../notifications/deprecated_config_file_used_email.txt | 4 ++-- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 47c85a7591f..57dfd7778fe 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -164,12 +164,11 @@ class DeprecatedConfigFileSiteNotification(SiteNotification): failure_message = ( "Your project '{{ object.slug }}' doesn't have a " - '.readthedocs.yaml ' - "configuration file. " + ".readthedocs.yaml configuration file. " "Configuration files will soon be required by projects, " "and will no longer be optional. " - "Make sure to create one for your project to ensure your project continues " - "building successfully." + 'Read our blog post to create one' # noqa + "to ensure your project continues building successfully." ) failure_level = WARNING_PERSISTENT diff --git a/readthedocs/templates/builds/build_detail.html b/readthedocs/templates/builds/build_detail.html index 4f12852835a..60095e58a15 100644 --- a/readthedocs/templates/builds/build_detail.html +++ b/readthedocs/templates/builds/build_detail.html @@ -166,10 +166,11 @@ {% if build.finished and build.deprecated_config_used %}

- {% blocktrans trimmed with config_file_link="https://docs.readthedocs.io/page/config-file/v2.html" %} + {% blocktrans trimmed with config_file_link="https://blog.readthedocs.com/migrate-configuration-v2/" %} Your builds will stop working soon!
Configuration files will soon be required by projects, and will no longer be optional. - Add a .readthedocs.yaml config file to your project to ensure your project continues building successfully. + Read our blog post to create one + to ensure your project continues building successfully. {% endblocktrans %}

diff --git a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html index 2092399b5fc..3f50f52c53c 100644 --- a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html +++ b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html @@ -1,10 +1,13 @@ {% extends "core/email/common.html" %} {% block content %} -Your project "{{ project.slug }}" is not using a .readthedocs.yaml configuration file and builds will stop working soon. +Your project "{{ project.slug }}" is not using a .readthedocs.yaml configuration file and builds will stop working soon. Configuration files will soon be required by projects, and will no longer be optional. You will need to add a configuration file to your project to ensure your project continues building successfully. +Read our blog post to create one to ensure your project continues building successfully. + + Get in touch with us via our support and let us know if you are unable to use a configuration file for any reason. {% endblock %} diff --git a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt index b93ed737c7d..5db0559babc 100644 --- a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt +++ b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt @@ -5,8 +5,8 @@ Your project "{{ project.slug }}" is not using a .readthedocs.yaml configuration Configuration files will soon be required by projects, and will no longer be optional. You will need to add a configuration file to your project to ensure your project continues building successfully. -For more information on our configuration file, see: -https://docs.readthedocs.io/en/stable/config-file/v2.html +For more information on how to create a required configuration file, see: +https://blog.readthedocs.com/migrate-configuration-v2/ Get in touch with us at {{ production_uri }}{% url 'support' %} and let us know if you are unable to use a configuration file for any reason. From acea9fc32bd42850946ffb5d1dae234fe42875a6 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 31 May 2023 11:59:10 +0200 Subject: [PATCH 09/31] Add more logging for this task --- readthedocs/projects/tasks/utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 57dfd7778fe..556619d19ba 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -226,6 +226,7 @@ def deprecated_config_file_used_notification(): ) projects = Project.objects.filter(slug__in=projects) + start_datetime = datetime.datetime.now() for i, project in enumerate(projects): if i % 500 == 0: @@ -249,6 +250,11 @@ def deprecated_config_file_used_notification(): ) n_email.send() + log.info( + "Finish sending deprecated config file notifications.", + notification_seconds=(datetime.datetime.now() - start_datetime).seconds, + ) + class BuildRequest(Request): From f4831d6c3fc597b0d9a73319ec5d9b8db1387ac5 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 31 May 2023 12:03:21 +0200 Subject: [PATCH 10/31] Space typo --- readthedocs/projects/tasks/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 556619d19ba..677b6e30772 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -167,7 +167,7 @@ class DeprecatedConfigFileSiteNotification(SiteNotification): ".readthedocs.yaml configuration file. " "Configuration files will soon be required by projects, " "and will no longer be optional. " - 'Read our blog post to create one' # noqa + 'Read our blog post to create one ' # noqa "to ensure your project continues building successfully." ) failure_level = WARNING_PERSISTENT From d96f6518c52c50f69d629df7ef6106d99871b50b Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 31 May 2023 12:33:01 +0200 Subject: [PATCH 11/31] Ignore projects that are potentially spam --- readthedocs/projects/tasks/utils.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 677b6e30772..5e86c3deb2b 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -3,7 +3,7 @@ import structlog from celery.worker.request import Request -from django.db.models import Q +from django.db.models import Q, Sum from django.utils import timezone from django.utils.translation import gettext_lazy as _ from messages_extends.constants import WARNING_PERSISTENT @@ -194,12 +194,21 @@ def deprecated_config_file_used_notification(): This is a scheduled task to be executed on the webs. Note the code uses `.iterator` and `.only` to avoid killing the db with this query. + Besdies, it excludes projects with enough spam score to be skipped. """ + # Skip projects with a spam score bigger than this value. + # Currently, this gives us ~250k in total (from ~550k we have in our database) + spam_score = 300 + projects = set() - # NOTE: we could skip the projects with a spam score > 150, - # to reduce the amount of email/onsite notifications we send start_datetime = datetime.datetime.now() - for project in Project.objects.all().only("slug", "default_version").iterator(): + queryset = ( + Project.objects.exclude(users__profile__banned=True) + .annotate(spam_score=Sum("spam_rules__value")) + .filter(Q(spam_score__gte=1, spam_score__lt=spam_score) | Q(is_spam=False)) + .only("slug", "default_version") + ) + for project in queryset.iterator(): # NOTE: instead of iterating over all the active versions, # we can only consider the default one for version in ( From 400214f7468bb6f4fdf3e7d8171ecf08fc255731 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 31 May 2023 12:36:48 +0200 Subject: [PATCH 12/31] Order queryset by PK so we can track it Also, add log for current project in case we need to recover from that task. --- readthedocs/projects/tasks/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 5e86c3deb2b..44e3b41054c 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -234,14 +234,16 @@ def deprecated_config_file_used_notification(): projects=n_projects, ) - projects = Project.objects.filter(slug__in=projects) + projects = Project.objects.filter(slug__in=projects).order_by("id") start_datetime = datetime.datetime.now() - for i, project in enumerate(projects): + for i, project in enumerate(projects.iterator()): if i % 500 == 0: log.info( "Sending deprecated config file notifications.", progress=f"{i}/{n_projects}", + current_project_pk=project.pk, + current_project_slug=project.slug, ) users = AdminPermission.owners(project) From 50d10bd90bd84eefa22556a5514c3dea7a793d7e Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 31 May 2023 15:25:03 +0200 Subject: [PATCH 13/31] Improve query a little more --- readthedocs/projects/tasks/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 44e3b41054c..26d3aed7e75 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -205,7 +205,7 @@ def deprecated_config_file_used_notification(): queryset = ( Project.objects.exclude(users__profile__banned=True) .annotate(spam_score=Sum("spam_rules__value")) - .filter(Q(spam_score__gte=1, spam_score__lt=spam_score) | Q(is_spam=False)) + .filter(Q(spam_score__lt=spam_score) | Q(is_spam=False)) .only("slug", "default_version") ) for project in queryset.iterator(): From 3ba5cbbe79db89cf319c8cf5db6129f489b8e4f1 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 31 May 2023 15:46:48 +0200 Subject: [PATCH 14/31] Make the query to work on .com as well --- readthedocs/projects/tasks/utils.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 26d3aed7e75..7a82a81100d 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -3,6 +3,7 @@ import structlog from celery.worker.request import Request +from django.conf import settings from django.db.models import Q, Sum from django.utils import timezone from django.utils.translation import gettext_lazy as _ @@ -202,12 +203,14 @@ def deprecated_config_file_used_notification(): projects = set() start_datetime = datetime.datetime.now() - queryset = ( - Project.objects.exclude(users__profile__banned=True) - .annotate(spam_score=Sum("spam_rules__value")) - .filter(Q(spam_score__lt=spam_score) | Q(is_spam=False)) - .only("slug", "default_version") - ) + queryset = Project.objects.exclude(users__profile__banned=True) + if not settings.ALLOW_PRIVATE_REPOS: + # Take into account spam score on community + queryset = queryset.annotate(spam_score=Sum("spam_rules__value")).filter( + Q(spam_score__lt=spam_score) | Q(is_spam=False) + ) + queryset = queryset.only("slug", "default_version") + for project in queryset.iterator(): # NOTE: instead of iterating over all the active versions, # we can only consider the default one From 8889103401c4e95a4799fb415a7cdbdb2f425412 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 1 Jun 2023 09:19:11 +0200 Subject: [PATCH 15/31] Query only active subscriptions on .com --- readthedocs/projects/tasks/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 7a82a81100d..72ceb40f98e 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -204,7 +204,10 @@ def deprecated_config_file_used_notification(): projects = set() start_datetime = datetime.datetime.now() queryset = Project.objects.exclude(users__profile__banned=True) - if not settings.ALLOW_PRIVATE_REPOS: + if settings.ALLOW_PRIVATE_REPOS: + # Only send emails to active customers + queryset = queryset.filter(organizations__subscription__status="active") + else: # Take into account spam score on community queryset = queryset.annotate(spam_score=Sum("spam_rules__value")).filter( Q(spam_score__lt=spam_score) | Q(is_spam=False) From f85bf2cf1401536d6f80a7cb0f6f02dcc80bcbd9 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 1 Jun 2023 09:38:31 +0200 Subject: [PATCH 16/31] Consistency on naming --- readthedocs/projects/tasks/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 72ceb40f98e..8e9ee48633a 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -240,9 +240,9 @@ def deprecated_config_file_used_notification(): projects=n_projects, ) - projects = Project.objects.filter(slug__in=projects).order_by("id") + queryset = Project.objects.filter(slug__in=projects).order_by("id") start_datetime = datetime.datetime.now() - for i, project in enumerate(projects.iterator()): + for i, project in enumerate(queryset.iterator()): if i % 500 == 0: log.info( From 27e08abdb810018398d7316d57d9e73492462e29 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 1 Jun 2023 09:39:13 +0200 Subject: [PATCH 17/31] Only check for `Project.default_version` --- readthedocs/projects/tasks/utils.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 8e9ee48633a..ff73d4a009d 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -215,15 +215,11 @@ def deprecated_config_file_used_notification(): queryset = queryset.only("slug", "default_version") for project in queryset.iterator(): - # NOTE: instead of iterating over all the active versions, - # we can only consider the default one + # Only check for the default version because if the project is using tags + # they won't be able to update those and we will send them emails forever. + # We can update this query if we consider later. for version in ( - project.versions.filter(Q(active=True) | Q(slug=project.default_version)) - .order_by("-modified") - # Add a limit of 15 to protect ourselves against projects with - # hundred of active versions - .only("id")[:15] - .iterator() + project.versions.filter(slug=project.default_version).only("id").iterator() ): build = version.builds.filter(success=True).only("_config").first() if build and build.deprecated_config_used(): From aa916513b31d2348fb28cdff03b9f556be304466 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 1 Jun 2023 09:42:21 +0200 Subject: [PATCH 18/31] Log progress while iterating to know it's moving --- readthedocs/projects/tasks/utils.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index ff73d4a009d..548cbe39d91 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -212,9 +212,19 @@ def deprecated_config_file_used_notification(): queryset = queryset.annotate(spam_score=Sum("spam_rules__value")).filter( Q(spam_score__lt=spam_score) | Q(is_spam=False) ) - queryset = queryset.only("slug", "default_version") + queryset = queryset.only("slug", "default_version").order_by("id") + n_projects = queryset.count() + + for i, project in enumerate(queryset.iterator()): + + if i % 500 == 0: + log.info( + "Finding projects without a configuration file.", + progress=f"{i}/{n_projects}", + current_project_pk=project.pk, + current_project_slug=project.slug, + ) - for project in queryset.iterator(): # Only check for the default version because if the project is using tags # they won't be able to update those and we will send them emails forever. # We can update this query if we consider later. From 0a17b24d90a6b065b7b2fad5c18ef8473d75e95e Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 1 Jun 2023 09:54:12 +0200 Subject: [PATCH 19/31] Simplify versions query --- readthedocs/projects/tasks/utils.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 548cbe39d91..2fd2f71277a 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -228,16 +228,18 @@ def deprecated_config_file_used_notification(): # Only check for the default version because if the project is using tags # they won't be able to update those and we will send them emails forever. # We can update this query if we consider later. - for version in ( - project.versions.filter(slug=project.default_version).only("id").iterator() - ): - build = version.builds.filter(success=True).only("_config").first() - if build and build.deprecated_config_used(): + version = ( + project.versions.filter(slug=project.default_version).only("id").first() + ) + if version: + build = ( + version.builds.filter(success=True) + .only("_config") + .order_by("-date") + .first() + ) + if build and not build.using_latest_config(): projects.add(project.slug) - # Do not continue iterating over the - # other versions when we know this project - # will get the notification - break n_projects = len(projects) log.info( From c294d76a80816d43ab165400623d9347c5c44b2a Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 1 Jun 2023 09:54:36 +0200 Subject: [PATCH 20/31] More logging to the progress --- readthedocs/projects/tasks/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 2fd2f71277a..4613a800eab 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -223,6 +223,8 @@ def deprecated_config_file_used_notification(): progress=f"{i}/{n_projects}", current_project_pk=project.pk, current_project_slug=project.slug, + projects_found=len(projects), + time_elapsed=(datetime.datetime.now() - start_datetime).seconds, ) # Only check for the default version because if the project is using tags From e2a78fef2e900486f3fb71a150672722aa79c817 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 1 Jun 2023 10:25:53 +0200 Subject: [PATCH 21/31] Send only one notification per user The notification will include all the projects the user is admin that are affected by this deprecation. Users will receive at most one notification per week. --- readthedocs/projects/tasks/utils.py | 56 +++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 4613a800eab..b6087552a23 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -4,6 +4,7 @@ import structlog from celery.worker.request import Request from django.conf import settings +from django.contrib.auth.models import User from django.db.models import Q, Sum from django.utils import timezone from django.utils.translation import gettext_lazy as _ @@ -163,6 +164,8 @@ def send_external_build_status(version_type, build_pk, commit, status): class DeprecatedConfigFileSiteNotification(SiteNotification): + # TODO: mention all the project slugs here + # Maybe trim them to up to 5 projects to avoid sending a huge blob of text failure_message = ( "Your project '{{ object.slug }}' doesn't have a " ".readthedocs.yaml configuration file. " @@ -216,7 +219,6 @@ def deprecated_config_file_used_notification(): n_projects = queryset.count() for i, project in enumerate(queryset.iterator()): - if i % 500 == 0: log.info( "Finding projects without a configuration file.", @@ -250,32 +252,56 @@ def deprecated_config_file_used_notification(): projects=n_projects, ) + # Store all the users we want to contact + users = set() + queryset = Project.objects.filter(slug__in=projects).order_by("id") start_datetime = datetime.datetime.now() for i, project in enumerate(queryset.iterator()): - if i % 500 == 0: log.info( - "Sending deprecated config file notifications.", + "Querying all the users we want to contact.", progress=f"{i}/{n_projects}", current_project_pk=project.pk, current_project_slug=project.slug, ) - users = AdminPermission.owners(project) - for user in users: - n_site = DeprecatedConfigFileSiteNotification( - user=user, - context_object=project, - success=False, - ) - n_site.send() + users.update(AdminPermission.owners(project).values_list("username", flat=True)) - n_email = DeprecatedConfigFileEmailNotification( - user=user, - context_object=project, + # Only send 1 email per user, + # even if that user has multiple projects without a configuration file. + # The notification will mention all the projects. + queryset = User.objects.filter(username__in=users, profile__banned=False).order_by( + "id" + ) + n_users = queryset.count() + for i, user in enumerate(queryset.iterator()): + if i % 500 == 0: + log.info( + "Querying all the users we want to contact.", + progress=f"{i}/{n_users}", + current_user_pk=user.pk, + current_user_username=user.username, ) - n_email.send() + + # All the projects for this user that don't have a configuration file + user_projects = ( + AdminPermission.projects(user, admin=True) + .filter(slug__in=projects) + .only("slug") + ) + n_site = DeprecatedConfigFileSiteNotification( + user=user, + context_object=user_projects, + success=False, + ) + n_site.send() + + n_email = DeprecatedConfigFileEmailNotification( + user=user, + context_object=user_projects, + ) + n_email.send() log.info( "Finish sending deprecated config file notifications.", From 094bf89d02e88025fd0389e10ae996c066cf72ae Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 1 Jun 2023 10:26:47 +0200 Subject: [PATCH 22/31] Modify email template to include all the projects and dates --- .../deprecated_config_file_used_email.txt | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt index 5db0559babc..99056d5e05a 100644 --- a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt +++ b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt @@ -1,9 +1,20 @@ {% extends "core/email/common.txt" %} {% block content %} -Your project "{{ project.slug }}" is not using a .readthedocs.yaml configuration file and builds will stop working soon. +The Read the Docs build system will start requiring a configuration file v2 (.readthedocs.yaml) starting on September 4, 2023. +We are scheduling brownout days to provide extra reminders by failing build without a configuration file v2 during some hours before the final day. +Keep these dates in mind to avoid unexpected behaviours: -Configuration files will soon be required by projects, and will no longer be optional. -You will need to add a configuration file to your project to ensure your project continues building successfully. +* Monday, July 24, 2023: Do the first brownout (temporarily enforce this deprecation) for 12 hours. +* Monday, August 7, 2023: Do a second brownout (temporarily enforce this deprecation) for 24 hours. +* Monday, September 4, 2023: Fully remove support for building documentation without configuration file v2. + +We have identified the following projects where you are admin are impacted by this deprecation: + +{% for project in object %} +* {{ project.slug }} +{% endfor %} + +You require to add a configuration file to your projects to ensure they continues building successfully and stop receiving these notifications. For more information on how to create a required configuration file, see: https://blog.readthedocs.com/migrate-configuration-v2/ From e244371f279f6497f6d125ed9f07ef31943220fe Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 1 Jun 2023 10:32:01 +0200 Subject: [PATCH 23/31] Typo --- readthedocs/projects/tasks/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index b6087552a23..2606299c10b 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -242,7 +242,7 @@ def deprecated_config_file_used_notification(): .order_by("-date") .first() ) - if build and not build.using_latest_config(): + if build and build.deprecated_config_used(): projects.add(project.slug) n_projects = len(projects) From 98941e51a79edcc8504473f1f19bbecc28889d30 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 1 Jun 2023 10:35:06 +0200 Subject: [PATCH 24/31] Improve logging --- readthedocs/projects/tasks/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 2606299c10b..459db1be9dd 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -256,7 +256,6 @@ def deprecated_config_file_used_notification(): users = set() queryset = Project.objects.filter(slug__in=projects).order_by("id") - start_datetime = datetime.datetime.now() for i, project in enumerate(queryset.iterator()): if i % 500 == 0: log.info( @@ -264,6 +263,7 @@ def deprecated_config_file_used_notification(): progress=f"{i}/{n_projects}", current_project_pk=project.pk, current_project_slug=project.slug, + time_elapsed=(datetime.datetime.now() - start_datetime).seconds, ) users.update(AdminPermission.owners(project).values_list("username", flat=True)) @@ -282,6 +282,7 @@ def deprecated_config_file_used_notification(): progress=f"{i}/{n_users}", current_user_pk=user.pk, current_user_username=user.username, + time_elapsed=(datetime.datetime.now() - start_datetime).seconds, ) # All the projects for this user that don't have a configuration file @@ -305,7 +306,7 @@ def deprecated_config_file_used_notification(): log.info( "Finish sending deprecated config file notifications.", - notification_seconds=(datetime.datetime.now() - start_datetime).seconds, + time_elapsed=(datetime.datetime.now() - start_datetime).seconds, ) From 4fb54f0ee74bd65d54e3a5527fc07a7ba67d5f8c Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 1 Jun 2023 11:06:45 +0200 Subject: [PATCH 25/31] Keep adding logging :) --- readthedocs/projects/tasks/utils.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 459db1be9dd..7e63f0faf43 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -245,16 +245,10 @@ def deprecated_config_file_used_notification(): if build and build.deprecated_config_used(): projects.add(project.slug) - n_projects = len(projects) - log.info( - "Sending deprecated config file notification.", - query_seconds=(datetime.datetime.now() - start_datetime).seconds, - projects=n_projects, - ) - # Store all the users we want to contact users = set() + n_projects = len(projects) queryset = Project.objects.filter(slug__in=projects).order_by("id") for i, project in enumerate(queryset.iterator()): if i % 500 == 0: @@ -263,6 +257,7 @@ def deprecated_config_file_used_notification(): progress=f"{i}/{n_projects}", current_project_pk=project.pk, current_project_slug=project.slug, + users_found=len(users), time_elapsed=(datetime.datetime.now() - start_datetime).seconds, ) @@ -278,7 +273,7 @@ def deprecated_config_file_used_notification(): for i, user in enumerate(queryset.iterator()): if i % 500 == 0: log.info( - "Querying all the users we want to contact.", + "Sending deprecated config file notification to users.", progress=f"{i}/{n_users}", current_user_pk=user.pk, current_user_username=user.username, From 7bda672d8988f3ee9b914dc363e4848ba84c3dee Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 6 Jun 2023 08:34:50 +0200 Subject: [PATCH 26/31] Db query for active subscriptions on .com --- readthedocs/projects/tasks/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 7e63f0faf43..b20562e0133 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -8,6 +8,7 @@ from django.db.models import Q, Sum from django.utils import timezone from django.utils.translation import gettext_lazy as _ +from djstripe.enums import SubscriptionStatus from messages_extends.constants import WARNING_PERSISTENT from readthedocs.builds.constants import ( @@ -209,7 +210,9 @@ def deprecated_config_file_used_notification(): queryset = Project.objects.exclude(users__profile__banned=True) if settings.ALLOW_PRIVATE_REPOS: # Only send emails to active customers - queryset = queryset.filter(organizations__subscription__status="active") + queryset = queryset.filter( + organizations__stripe_subscription__status=SubscriptionStatus.active + ) else: # Take into account spam score on community queryset = queryset.annotate(spam_score=Sum("spam_rules__value")).filter( From e42e9c659008ba9c6837645e91d7558b9cfa7312 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 6 Jun 2023 08:35:09 +0200 Subject: [PATCH 27/31] Email subject --- readthedocs/projects/tasks/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index b20562e0133..71b4b1d862b 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -183,7 +183,7 @@ class DeprecatedConfigFileEmailNotification(Notification): app_templates = "projects" name = "deprecated_config_file_used" context_object_name = "project" - subject = "Add a configuration file to your project to continue building" + subject = "[Action required] Add a configuration file to your project to prevent build failure" level = REQUIREMENT def send(self): From 8dbbf6edb6b39118afdf65a3337b96ad91fbfe43 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 6 Jun 2023 09:01:39 +0200 Subject: [PATCH 28/31] Update onsite notification message --- readthedocs/projects/tasks/utils.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 71b4b1d862b..022d056f3c3 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -167,13 +167,12 @@ class DeprecatedConfigFileSiteNotification(SiteNotification): # TODO: mention all the project slugs here # Maybe trim them to up to 5 projects to avoid sending a huge blob of text - failure_message = ( - "Your project '{{ object.slug }}' doesn't have a " - ".readthedocs.yaml configuration file. " + failure_message = _( + 'Your project(s) "{{ project_slugs }}" don\'t have a configuration file. ' "Configuration files will soon be required by projects, " "and will no longer be optional. " 'Read our blog post to create one ' # noqa - "to ensure your project continues building successfully." + "and ensure your project continues building successfully." ) failure_level = WARNING_PERSISTENT @@ -289,9 +288,15 @@ def deprecated_config_file_used_notification(): .filter(slug__in=projects) .only("slug") ) + + user_project_slugs = ", ".join([p.slug for p in user_projects[:5]]) + if user_projects.count() > 5: + user_project_slugs += " and others..." + n_site = DeprecatedConfigFileSiteNotification( user=user, context_object=user_projects, + extra_context={"project_slugs": user_project_slugs}, success=False, ) n_site.send() From 136da2bb295c57434d6002594e89691c5012ae91 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 6 Jun 2023 09:01:58 +0200 Subject: [PATCH 29/31] Do not set emails just yet --- readthedocs/projects/tasks/utils.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 022d056f3c3..9d4abfb8f1a 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -301,11 +301,12 @@ def deprecated_config_file_used_notification(): ) n_site.send() - n_email = DeprecatedConfigFileEmailNotification( - user=user, - context_object=user_projects, - ) - n_email.send() + # TODO: uncomment this code when we are ready to send email notifications + # n_email = DeprecatedConfigFileEmailNotification( + # user=user, + # context_object=user_projects, + # ) + # n_email.send() log.info( "Finish sending deprecated config file notifications.", From 5d7f42c924c24f092ae8ad4bf8ef8fedfb26507f Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 6 Jun 2023 09:09:32 +0200 Subject: [PATCH 30/31] Minor updates --- readthedocs/projects/tasks/utils.py | 1 + readthedocs/templates/builds/build_detail.html | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/readthedocs/projects/tasks/utils.py b/readthedocs/projects/tasks/utils.py index 9d4abfb8f1a..d802883cfbb 100644 --- a/readthedocs/projects/tasks/utils.py +++ b/readthedocs/projects/tasks/utils.py @@ -305,6 +305,7 @@ def deprecated_config_file_used_notification(): # n_email = DeprecatedConfigFileEmailNotification( # user=user, # context_object=user_projects, + # extra_context={"project_slugs": user_project_slugs}, # ) # n_email.send() diff --git a/readthedocs/templates/builds/build_detail.html b/readthedocs/templates/builds/build_detail.html index 60095e58a15..17bf9424074 100644 --- a/readthedocs/templates/builds/build_detail.html +++ b/readthedocs/templates/builds/build_detail.html @@ -170,7 +170,7 @@ Your builds will stop working soon!
Configuration files will soon be required by projects, and will no longer be optional. Read our blog post to create one - to ensure your project continues building successfully. + and ensure your project continues building successfully. {% endblocktrans %}

From 0a947a19ad190d144e601413feb7efa585878bb5 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 6 Jun 2023 09:09:48 +0200 Subject: [PATCH 31/31] Update emails with new dates --- .../deprecated_config_file_used_email.html | 24 +++++++++++++++---- .../deprecated_config_file_used_email.txt | 9 +++---- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html index 3f50f52c53c..81778d139cb 100644 --- a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html +++ b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html @@ -1,12 +1,28 @@ {% extends "core/email/common.html" %} {% block content %} -Your project "{{ project.slug }}" is not using a .readthedocs.yaml configuration file and builds will stop working soon. +The Read the Docs build system will start requiring a configuration file v2 (.readthedocs.yaml) starting on September 25, 2023. +We are scheduling brownout days to provide extra reminders by failing build without a configuration file v2 during some hours before the final day. +Keep these dates in mind to avoid unexpected behaviours: -Configuration files will soon be required by projects, and will no longer be optional. -You will need to add a configuration file to your project to ensure your project continues building successfully. +
    +
  • Monday, July 24, 2023: Do the first brownout (temporarily enforce this deprecation) for 12 hours: 00:01 PST to 11:59 PST (noon)
  • +
  • Monday, August 14, 2023: Do a second brownout (temporarily enforce this deprecation) for 24 hours: 00:01 PST to 23:59 PST (midnight)
  • +
  • Monday, September 4, 2023 Do a third and final brownout (temporarily enforce this deprecation) for 48 hours: 00:01 PST to 23:59 PST (midnight)
  • +
  • Monday, September 25, 2023: Fully remove support for building documentation without configuration file v2.
  • +
-Read our blog post to create one to ensure your project continues building successfully. +We have identified the following projects where you are admin are impacted by this deprecation: +
    +{% for project in object %} +
  • {{ project.slug }}
  • +{% endfor %} +
+ +You require to add a configuration file to your projects to ensure they continues building successfully and stop receiving these notifications. + +For more information on how to create a required configuration file, +read our blog post Get in touch with us via our support and let us know if you are unable to use a configuration file for any reason. diff --git a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt index 99056d5e05a..31d7fb8734c 100644 --- a/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt +++ b/readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt @@ -1,12 +1,13 @@ {% extends "core/email/common.txt" %} {% block content %} -The Read the Docs build system will start requiring a configuration file v2 (.readthedocs.yaml) starting on September 4, 2023. +The Read the Docs build system will start requiring a configuration file v2 (.readthedocs.yaml) starting on September 25, 2023. We are scheduling brownout days to provide extra reminders by failing build without a configuration file v2 during some hours before the final day. Keep these dates in mind to avoid unexpected behaviours: -* Monday, July 24, 2023: Do the first brownout (temporarily enforce this deprecation) for 12 hours. -* Monday, August 7, 2023: Do a second brownout (temporarily enforce this deprecation) for 24 hours. -* Monday, September 4, 2023: Fully remove support for building documentation without configuration file v2. +* Monday, July 24, 2023: Do the first brownout (temporarily enforce this deprecation) for 12 hours: 00:01 PST to 11:59 PST (noon) +* Monday, August 14, 2023: Do a second brownout (temporarily enforce this deprecation) for 24 hours: 00:01 PST to 23:59 PST (midnight) +* Monday, September 4, 2023: Do a third and final brownout (temporarily enforce this deprecation) for 48 hours: 00:01 PST to 23:59 PST (midnight) +* Monday, September 25, 2023: Fully remove support for building documentation without configuration file v2. We have identified the following projects where you are admin are impacted by this deprecation: