diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 749d49c6..4b3547a0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -47,8 +47,5 @@ jobs: - name: Install dependencies run: just install - - name: Install pre-commit hooks - run: pre-commit install --install-hooks - - name: Run lint and type checks run: just lint diff --git a/README.md b/README.md index 17e94d18..d2da307b 100644 --- a/README.md +++ b/README.md @@ -95,9 +95,11 @@ The system is designed for graceful failure, not silent corruption. Unparseable ```bash python3 -m venv .venv source .venv/bin/activate -python3 -m pip install -r requirements.txt +just install ``` +`just install` installs the backend and frontend dependencies and registers the repository's `pre-commit` hooks, so `git commit` runs the configured lint and test hooks locally. + 1. Run `just dev` to start Django, Celery, Postgres, Redis, Qdrant, and Nginx. On the first run Docker builds the app image automatically. After that, `just dev` reuses the existing image so normal restarts are fast. If `.env` is missing, the `just` command copies `.env.example` automatically. 2. Run `just build` after changing `requirements.txt` or `docker/web/Dockerfile`. 3. For a fully fresh local stack after schema changes, run `just reset-volumes` before starting the containers again. This drops the Docker-backed Postgres, Redis, and Qdrant state so regenerated migrations apply cleanly. diff --git a/SESSION.md b/SESSION.md index e5657c34..3977f3f3 100644 --- a/SESSION.md +++ b/SESSION.md @@ -1,16 +1 @@ # Session Restore Point - -I’m checking how this repo wires the database for just migrate so I can tell you whether reset-volumes will actually drop the database you’re about to migrate. - -Searched for database configuration in settings files -Not necessarily. - -just reset-volumes runs docker compose down -v --remove-orphans, so it deletes the Docker named volumes from justfile:119 and justfile:159. In this repo that includes the Postgres volume from docker-compose.yml:35 and docker-compose.yml:79, but also Redis, Qdrant, and the frontend node_modules volume. - -The important detail is that just migrate is a local python3 manage.py migrate command from justfile:123, and the repo default .env.example points DATABASE_URL at SQLite in .env.example:5. So: - -If your current .env still uses sqlite:///db.sqlite3, just reset-volumes will not drop the database you are migrating. You would need to remove db.sqlite3 instead, then run just migrate. -If your current .env points at the Docker Postgres service, then yes, just reset-volumes will wipe that Postgres database. After that, bring services back up so Postgres exists again, then run just migrate. -So the short answer is: only use just reset-volumes if the database you want to drop is the Dockerized Postgres one. If you want, I can help you verify which DB your current .env is targeting before you do anything destructive. - -DATABASE_URL=sqlite:///db.sqlite3 diff --git a/core/api.py b/core/api.py index b8093372..92274493 100644 --- a/core/api.py +++ b/core/api.py @@ -117,6 +117,23 @@ response_only=True, ) +MASTODON_CREDENTIALS_RESPONSE_EXAMPLE = OpenApiExample( + "Mastodon Credentials Response", + value={ + "id": 1, + "project": 1, + "instance_url": "https://hachyderm.io", + "account_acct": "alice@hachyderm.io", + "is_active": True, + "has_stored_credential": True, + "last_verified_at": "2026-04-26T13:00:00Z", + "last_error": "", + "created_at": "2026-04-26T12:30:00Z", + "updated_at": "2026-04-26T13:00:00Z", + }, + response_only=True, +) + SOURCE_CONFIG_CREATE_REQUEST_EXAMPLE = OpenApiExample( "Create RSS Source Request", value={ @@ -157,6 +174,22 @@ request_only=True, ) +SOURCE_CONFIG_MASTODON_REQUEST_EXAMPLE = OpenApiExample( + "Create Mastodon Source Request", + value={ + "plugin_name": "mastodon", + "config": { + "instance_url": "https://hachyderm.io", + "hashtag": "platformengineering", + "include_replies": False, + "include_reblogs": True, + "max_statuses_per_fetch": 100, + }, + "is_active": True, + }, + request_only=True, +) + SOURCE_CONFIG_RESPONSE_EXAMPLE = OpenApiExample( "Source Configuration Response", value={ @@ -284,6 +317,17 @@ }, ) +MASTODON_CREDENTIALS_VERIFY_RESPONSE = inline_serializer( + name="MastodonCredentialsVerifyResponse", + fields={ + "status": serializers.CharField(), + "account_acct": serializers.CharField(allow_blank=True), + "instance_url": serializers.URLField(), + "last_verified_at": serializers.DateTimeField(allow_null=True), + "last_error": serializers.CharField(allow_blank=True), + }, +) + def build_success_response( response, description: str, examples: list[OpenApiExample] | None = None diff --git a/core/plugins/mastodon.py b/core/plugins/mastodon.py new file mode 100644 index 00000000..cdef11d2 --- /dev/null +++ b/core/plugins/mastodon.py @@ -0,0 +1,7 @@ +"""Compatibility wrapper for the Mastodon source plugin.""" + +from mastodon import Mastodon + +from ingestion.plugins.mastodon import MastodonSourcePlugin + +__all__ = ["Mastodon", "MastodonSourcePlugin"] diff --git a/core/tests/test_admin.py b/core/tests/test_admin.py index 71f96ad9..cd85b1f8 100644 --- a/core/tests/test_admin.py +++ b/core/tests/test_admin.py @@ -39,11 +39,19 @@ from projects.admin import ( BlueskyCredentialsAdmin, BlueskyCredentialsAdminForm, + MastodonCredentialsAdmin, + MastodonCredentialsAdminForm, ProjectConfigAdmin, SourceConfigAdmin, ) from projects.model_support import SourcePluginName -from projects.models import BlueskyCredentials, Project, ProjectConfig, SourceConfig +from projects.models import ( + BlueskyCredentials, + MastodonCredentials, + Project, + ProjectConfig, + SourceConfig, +) pytestmark = pytest.mark.django_db @@ -365,6 +373,82 @@ def test_verify_selected_bluesky_credentials_reports_failures( ) +def test_mastodon_credentials_admin_form_encrypts_access_token(source_admin_context): + form = MastodonCredentialsAdminForm( + data={ + "project": source_admin_context.project.id, + "instance_url": "https://hachyderm.io/@alice/", + "account_acct": "@Alice", + "credential_input": "access-token", + "is_active": True, + } + ) + + assert form.is_valid(), form.errors + credentials = form.save() + + assert credentials.instance_url == "https://hachyderm.io" + assert credentials.account_acct == "alice@hachyderm.io" + assert credentials.has_access_token() is True + assert credentials.get_access_token() == "access-token" + + +def test_verify_selected_mastodon_credentials_reports_success( + source_admin_context, mocker +): + credentials = MastodonCredentials.objects.create( + project=source_admin_context.project, + instance_url="https://hachyderm.io", + account_acct="alice@hachyderm.io", + access_token_encrypted="ciphertext", + ) + verify_mock = mocker.patch( + "core.plugins.mastodon.MastodonSourcePlugin.verify_credentials" + ) + admin_instance = MastodonCredentialsAdmin(MastodonCredentials, AdminSite()) + admin_instance.message_user = mocker.Mock() + + admin_instance.verify_selected_credentials( + request=SimpleNamespace(), + queryset=MastodonCredentials.objects.filter(pk=credentials.pk), + ) + + verify_mock.assert_called_once_with(credentials) + admin_instance.message_user.assert_called_once_with( + ANY, + "Credential verification passed for 1 account(s).", + messages.SUCCESS, + ) + + +def test_verify_selected_mastodon_credentials_reports_failures( + source_admin_context, mocker +): + credentials = MastodonCredentials.objects.create( + project=source_admin_context.project, + instance_url="https://hachyderm.io", + account_acct="alice@hachyderm.io", + access_token_encrypted="ciphertext", + ) + mocker.patch( + "core.plugins.mastodon.MastodonSourcePlugin.verify_credentials", + side_effect=RuntimeError("bad token"), + ) + admin_instance = MastodonCredentialsAdmin(MastodonCredentials, AdminSite()) + admin_instance.message_user = mocker.Mock() + + admin_instance.verify_selected_credentials( + request=SimpleNamespace(), + queryset=MastodonCredentials.objects.filter(pk=credentials.pk), + ) + + admin_instance.message_user.assert_called_once_with( + ANY, + "Credential verification failed for: Mastodon credentials for Admin Project: bad token", + messages.ERROR, + ) + + def test_ingestion_run_display_efficiency_renders_without_django6_format_error( source_admin_context, ): diff --git a/core/tests/test_api.py b/core/tests/test_api.py index c78f4a80..f8474db1 100644 --- a/core/tests/test_api.py +++ b/core/tests/test_api.py @@ -29,6 +29,7 @@ from projects.model_support import SourcePluginName from projects.models import ( BlueskyCredentials, + MastodonCredentials, Project, ProjectConfig, ProjectMembership, @@ -760,6 +761,81 @@ def test_verify_bluesky_credentials_surfaces_verification_errors( self.owner_project.id, ) + def test_verify_mastodon_credentials_requires_configured_project_credentials(self): + response = self.client.post( + reverse( + "v1:project-verify-mastodon-credentials", + kwargs={"id": self.owner_project.id}, + ), + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assert_standardized_validation_error( + response.json(), "mastodon_credentials" + ) + + @patch("core.plugins.mastodon.MastodonSourcePlugin.verify_credentials") + def test_verify_mastodon_credentials_verifies_project_account(self, verify_mock): + credentials = MastodonCredentials( + project=self.owner_project, + instance_url="https://hachyderm.io", + account_acct="alice@hachyderm.io", + ) + credentials.set_access_token("access-token") + credentials.save() + + response = self.client.post( + reverse( + "v1:project-verify-mastodon-credentials", + kwargs={"id": self.owner_project.id}, + ), + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + verify_mock.assert_called_once() + verified_credentials = verify_mock.call_args.args[0] + self.assertEqual(verified_credentials.id, credentials.id) + self.assertEqual(response.json()["status"], "verified") + self.assertEqual(response.json()["account_acct"], "alice@hachyderm.io") + self.assertEqual(response.json()["instance_url"], "https://hachyderm.io") + self.assertEqual(response.json()["last_error"], "") + + @patch("core.api.logger.exception") + @patch( + "core.plugins.mastodon.MastodonSourcePlugin.verify_credentials", + side_effect=RuntimeError("bad token"), + ) + def test_verify_mastodon_credentials_surfaces_verification_errors( + self, _verify_mock, logger_exception_mock + ): + credentials = MastodonCredentials( + project=self.owner_project, + instance_url="https://hachyderm.io", + account_acct="alice@hachyderm.io", + ) + credentials.set_access_token("access-token") + credentials.save() + + response = self.client.post( + reverse( + "v1:project-verify-mastodon-credentials", + kwargs={"id": self.owner_project.id}, + ), + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assert_standardized_validation_error( + response.json(), "mastodon_credentials" + ) + self.assertNotIn("bad token", str(response.json())) + logger_exception_mock.assert_called_once_with( + "Mastodon credential verification failed for project id=%s", + self.owner_project.id, + ) + @patch("core.signals.queue_topic_centroid_recompute") def test_feedback_create_assigns_current_user(self, queue_centroid_mock): response = self.client.post( @@ -918,6 +994,10 @@ def test_authenticated_nested_list_endpoints_smoke(self): "v1:project-bluesky-credentials-list", kwargs={"project_id": self.owner_project.id}, ), + reverse( + "v1:project-mastodon-credentials-list", + kwargs={"project_id": self.owner_project.id}, + ), reverse( "v1:project-intake-allowlist-list", kwargs={"project_id": self.owner_project.id}, diff --git a/core/tests/test_mastodon.py b/core/tests/test_mastodon.py new file mode 100644 index 00000000..df847aad --- /dev/null +++ b/core/tests/test_mastodon.py @@ -0,0 +1,301 @@ +from datetime import UTC, datetime, timedelta +from types import SimpleNamespace + +import pytest + +from core.models import Entity +from core.plugins.mastodon import MastodonSourcePlugin +from projects.model_support import SourcePluginName +from projects.models import MastodonCredentials, Project, SourceConfig + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def mastodon_context(): + project = Project.objects.create(name="Mastodon Project", topic_description="Infra") + entity = Entity.objects.create( + project=project, + name="Alice", + type="person", + mastodon_handle="@alice@hachyderm.io", + website_url="https://example.com/company", + ) + source_config = SourceConfig.objects.create( + project=project, + plugin_name=SourcePluginName.MASTODON, + config={ + "instance_url": "https://hachyderm.io", + "hashtag": "platformengineering", + }, + ) + return SimpleNamespace(project=project, entity=entity, source_config=source_config) + + +def test_mastodon_validate_config_normalizes_defaults_and_rejects_invalid_values(): + assert MastodonSourcePlugin.validate_config( + {"instance_url": "https://hachyderm.io/", "hashtag": "#PlatformEngineering"} + ) == { + "instance_url": "https://hachyderm.io", + "hashtag": "platformengineering", + "max_statuses_per_fetch": 100, + "include_replies": False, + "include_reblogs": True, + } + + assert MastodonSourcePlugin.validate_config( + { + "instance_url": "https://mastodon.social", + "account_acct": "@Alice", + "max_statuses_per_fetch": "5", + "include_replies": True, + "include_reblogs": False, + } + ) == { + "instance_url": "https://mastodon.social", + "account_acct": "alice@mastodon.social", + "max_statuses_per_fetch": 5, + "include_replies": True, + "include_reblogs": False, + } + + assert MastodonSourcePlugin.validate_config( + {"instance_url": "https://hachyderm.io", "list_id": "42"} + ) == { + "instance_url": "https://hachyderm.io", + "list_id": 42, + "max_statuses_per_fetch": 100, + "include_replies": False, + "include_reblogs": True, + } + + with pytest.raises(ValueError, match="Provide exactly one"): + MastodonSourcePlugin.validate_config({"instance_url": "https://hachyderm.io"}) + + with pytest.raises(ValueError, match="Provide exactly one"): + MastodonSourcePlugin.validate_config( + {"hashtag": "ai", "account_acct": "alice@hachyderm.io"} + ) + + with pytest.raises( + ValueError, match="max_statuses_per_fetch must be a positive integer" + ): + MastodonSourcePlugin.validate_config( + {"hashtag": "ai", "max_statuses_per_fetch": 0} + ) + + with pytest.raises(ValueError, match="include_replies must be a boolean"): + MastodonSourcePlugin.validate_config( + {"hashtag": "ai", "include_replies": "yes"} + ) + + +def test_mastodon_fetch_new_content_prefers_card_urls_and_dedupes_statuses( + mastodon_context, mocker +): + plugin = MastodonSourcePlugin(mastodon_context.source_config) + now = datetime.now(tz=UTC) + old_status = { + "uri": "https://hachyderm.io/users/alice/statuses/old", + "url": "https://hachyderm.io/@alice/old", + "created_at": now - timedelta(days=2), + "account": { + "acct": "alice", + "username": "alice", + "display_name": "Alice Example", + "url": "https://hachyderm.io/@alice", + }, + "content": "

Old post

", + "replies_count": 0, + "reblogs_count": 0, + "favourites_count": 0, + } + fresh_status = { + "uri": "https://hachyderm.io/users/alice/statuses/fresh", + "url": "https://hachyderm.io/@alice/fresh", + "created_at": now, + "account": { + "acct": "alice", + "username": "alice", + "display_name": "Alice Example", + "url": "https://hachyderm.io/@alice", + }, + "content": "

Check this out

", + "card": { + "url": "https://example.com/article", + "title": "Linked article", + }, + "replies_count": 1, + "reblogs_count": 2, + "favourites_count": 3, + } + duplicate_status = dict(fresh_status) + mocker.patch.object( + MastodonSourcePlugin, + "_get_statuses", + return_value=[old_status, fresh_status, duplicate_status], + ) + + items = plugin.fetch_new_content(since=now - timedelta(hours=1)) + + assert len(items) == 1 + assert items[0].url == "https://example.com/article" + assert items[0].title == "Linked article" + assert items[0].author == "Alice Example" + assert items[0].content_text == "Check this out" + assert items[0].source_plugin == SourcePluginName.MASTODON + assert items[0].source_metadata == { + "author_acct": "alice@hachyderm.io", + "author_display_name": "Alice Example", + "embedded_url": "https://example.com/article", + "instance_url": "https://hachyderm.io", + "favorite_count": 3, + "reblog_count": 2, + "reply_count": 1, + "status_uri": "https://hachyderm.io/users/alice/statuses/fresh", + "status_url": "https://hachyderm.io/@alice/fresh", + } + + +def test_mastodon_match_entity_for_item_uses_mastodon_handle(mastodon_context): + plugin = MastodonSourcePlugin(mastodon_context.source_config) + + result = plugin.match_entity_for_item( + SimpleNamespace( + url="https://irrelevant.example.com/article", + source_metadata={"author_acct": "Alice@Hachyderm.io"}, + ) + ) + + assert result == mastodon_context.entity + + +def test_mastodon_health_check_queries_configured_hashtag_endpoint( + mastodon_context, mocker +): + client = SimpleNamespace( + timeline_hashtag=mocker.Mock(return_value=[]), + ratelimit_remaining=None, + ratelimit_reset=None, + ) + mocker.patch.object(MastodonSourcePlugin, "_client", return_value=client) + + plugin = MastodonSourcePlugin(mastodon_context.source_config) + + assert plugin.health_check() is True + client.timeline_hashtag.assert_called_once_with("platformengineering", limit=1) + + +def test_mastodon_get_statuses_uses_account_timeline_lookup(mastodon_context, mocker): + mastodon_context.source_config.config = { + "instance_url": "https://hachyderm.io", + "account_acct": "alice@hachyderm.io", + "include_replies": False, + "include_reblogs": True, + "max_statuses_per_fetch": 100, + } + client = SimpleNamespace( + account_lookup=mocker.Mock(return_value={"id": 7}), + account_statuses=mocker.Mock(return_value=[]), + ratelimit_remaining=None, + ratelimit_reset=None, + ) + mocker.patch.object(MastodonSourcePlugin, "_client", return_value=client) + + plugin = MastodonSourcePlugin(mastodon_context.source_config) + plugin._get_statuses() + + client.account_lookup.assert_called_once_with("alice@hachyderm.io") + client.account_statuses.assert_called_once_with( + 7, + limit=100, + exclude_replies=True, + exclude_reblogs=False, + ) + + +def test_mastodon_get_statuses_uses_list_timeline(mastodon_context, mocker): + mastodon_context.source_config.config = { + "instance_url": "https://hachyderm.io", + "list_id": 42, + "include_replies": False, + "include_reblogs": True, + "max_statuses_per_fetch": 100, + } + client = SimpleNamespace( + timeline_list=mocker.Mock(return_value=[]), + ratelimit_remaining=None, + ratelimit_reset=None, + ) + mocker.patch.object(MastodonSourcePlugin, "_client", return_value=client) + + plugin = MastodonSourcePlugin(mastodon_context.source_config) + plugin._get_statuses() + + client.timeline_list.assert_called_once_with(42, limit=100) + + +def test_mastodon_credentials_encrypt_token_and_normalize_fields(mastodon_context): + credentials = MastodonCredentials( + project=mastodon_context.project, + instance_url="https://hachyderm.io/@alice/", + account_acct="@Alice", + ) + credentials.set_access_token("access-token") + credentials.save() + credentials.refresh_from_db() + + assert credentials.instance_url == "https://hachyderm.io" + assert credentials.account_acct == "alice@hachyderm.io" + assert credentials.access_token_encrypted != "access-token" + assert credentials.get_access_token() == "access-token" + + +def test_mastodon_client_uses_authenticated_project_credentials( + mastodon_context, mocker +): + credentials = MastodonCredentials( + project=mastodon_context.project, + instance_url="https://hachyderm.io", + account_acct="alice@hachyderm.io", + ) + credentials.set_access_token("access-token") + credentials.save() + client = mocker.Mock() + mastodon_cls = mocker.patch("core.plugins.mastodon.Mastodon", return_value=client) + + plugin = MastodonSourcePlugin(mastodon_context.source_config) + + assert plugin._client() == client + mastodon_cls.assert_called_once_with( + access_token="access-token", + api_base_url="https://hachyderm.io", + ) + + +def test_mastodon_verify_credentials_updates_verified_account(mastodon_context, mocker): + credentials = MastodonCredentials( + project=mastodon_context.project, + instance_url="https://hachyderm.io", + ) + credentials.set_access_token("access-token") + credentials.save() + client = mocker.Mock() + client.account_verify_credentials.return_value = { + "acct": "alice", + "username": "alice", + "url": "https://hachyderm.io/@alice", + } + mastodon_cls = mocker.patch("core.plugins.mastodon.Mastodon", return_value=client) + + MastodonSourcePlugin.verify_credentials(credentials) + + mastodon_cls.assert_called_once_with( + access_token="access-token", + api_base_url="https://hachyderm.io", + ) + client.account_verify_credentials.assert_called_once_with() + credentials.refresh_from_db() + assert credentials.account_acct == "alice@hachyderm.io" + assert credentials.last_error == "" + assert credentials.last_verified_at is not None diff --git a/core/tests/test_serializers.py b/core/tests/test_serializers.py index c627116e..8d8c8eeb 100644 --- a/core/tests/test_serializers.py +++ b/core/tests/test_serializers.py @@ -19,7 +19,11 @@ ) from projects.model_support import SourcePluginName from projects.models import Project, ProjectMembership, ProjectRole, SourceConfig -from projects.serializers import ProjectSerializer, SourceConfigSerializer +from projects.serializers import ( + MastodonCredentialsSerializer, + ProjectSerializer, + SourceConfigSerializer, +) pytestmark = pytest.mark.django_db @@ -295,6 +299,57 @@ def test_source_config_serializer_normalizes_bluesky_author_handle_config( } +def test_source_config_serializer_normalizes_mastodon_hashtag_config( + serializer_context, +): + serializer = SourceConfigSerializer( + data={ + "plugin_name": SourcePluginName.MASTODON, + "config": { + "instance_url": "https://hachyderm.io/", + "hashtag": "#PlatformEngineering", + }, + "is_active": True, + }, + context={ + "request": _request_for(serializer_context.user), + "project": serializer_context.project, + }, + ) + + assert serializer.is_valid(), serializer.errors + assert serializer.validated_data["config"] == { + "instance_url": "https://hachyderm.io", + "hashtag": "platformengineering", + "include_replies": False, + "include_reblogs": True, + "max_statuses_per_fetch": 100, + } + + +def test_mastodon_credentials_serializer_encrypts_access_token(serializer_context): + serializer = MastodonCredentialsSerializer( + data={ + "instance_url": "https://hachyderm.io/", + "account_acct": "@Alice", + "access_token": "secret-token", + "is_active": True, + }, + context={ + "request": _request_for(serializer_context.user), + "project": serializer_context.project, + }, + ) + + assert serializer.is_valid(), serializer.errors + credentials = serializer.save(project=serializer_context.project) + + assert credentials.instance_url == "https://hachyderm.io" + assert credentials.account_acct == "alice@hachyderm.io" + assert credentials.has_stored_credential() is True + assert credentials.get_access_token() == "secret-token" + + def test_entity_serializer_filters_project_queryset_to_request_user(serializer_context): serializer = EntitySerializer( context={"request": _request_for(serializer_context.user)} diff --git a/core/tests/test_tasks.py b/core/tests/test_tasks.py index 4a026d22..b57e7cf3 100644 --- a/core/tests/test_tasks.py +++ b/core/tests/test_tasks.py @@ -262,6 +262,60 @@ def test_ingest_source_config_deduplicates_bluesky_posts_by_post_uri( process_content_delay_mock.assert_not_called() +def test_ingest_source_config_deduplicates_mastodon_statuses_by_status_uri( + source_plugin_context, mocker +): + upsert_embedding_mock = mocker.patch("core.tasks.upsert_content_embedding") + process_content_delay_mock = mocker.patch("core.tasks.process_content.delay") + source_config = SourceConfig.objects.create( + project=source_plugin_context.project, + plugin_name=SourcePluginName.MASTODON, + config={ + "instance_url": "https://hachyderm.io", + "hashtag": "platformengineering", + }, + ) + Content.objects.create( + project=source_plugin_context.project, + entity=source_plugin_context.entity, + url="https://example.com/existing-article", + title="Existing Mastodon Status", + author="Alice Example", + source_plugin=SourcePluginName.MASTODON, + published_date="2026-04-20T12:00:00Z", + content_text="Existing content", + source_metadata={ + "status_uri": "https://hachyderm.io/users/alice/statuses/abc123" + }, + ) + plugin = SimpleNamespace( + fetch_new_content=lambda since: [ + SimpleNamespace( + url="https://example.com/new-canonical-url", + title="Duplicate Mastodon Status", + author="Alice Example", + published_date=datetime(2026, 4, 20, 12, 0, tzinfo=timezone.utc), + content_text="Duplicate content", + source_plugin=SourcePluginName.MASTODON, + source_metadata={ + "author_acct": "alice@hachyderm.io", + "status_uri": "https://hachyderm.io/users/alice/statuses/abc123", + }, + ) + ], + match_entity_for_item=lambda item: source_plugin_context.entity, + ) + mocker.patch("ingestion.tasks.get_plugin_for_source_config", return_value=plugin) + + items_fetched, items_ingested = _ingest_source_config(source_config) + + assert items_fetched == 1 + assert items_ingested == 0 + assert Content.objects.filter(project=source_plugin_context.project).count() == 1 + upsert_embedding_mock.assert_not_called() + process_content_delay_mock.assert_not_called() + + def test_run_all_ingestions_enqueues_active_source_configs( source_plugin_context, mocker ): diff --git a/frontend/src/app/admin/sources/__tests__/page.test.tsx b/frontend/src/app/admin/sources/__tests__/page.test.tsx index 34c0b563..d4501490 100644 --- a/frontend/src/app/admin/sources/__tests__/page.test.tsx +++ b/frontend/src/app/admin/sources/__tests__/page.test.tsx @@ -6,6 +6,7 @@ import type { BlueskyCredentials, IngestionRun, IntakeAllowlistEntry, + MastodonCredentials, NewsletterIntake, Project, SourceConfig, @@ -15,6 +16,7 @@ const { getProjectBlueskyCredentialsMock, getProjectIngestionRunsMock, getProjectIntakeAllowlistMock, + getProjectMastodonCredentialsMock, getProjectNewsletterIntakesMock, getProjectsMock, getProjectSourceConfigsMock, @@ -23,6 +25,7 @@ const { getProjectBlueskyCredentialsMock: vi.fn(), getProjectIngestionRunsMock: vi.fn(), getProjectIntakeAllowlistMock: vi.fn(), + getProjectMastodonCredentialsMock: vi.fn(), getProjectNewsletterIntakesMock: vi.fn(), getProjectsMock: vi.fn(), getProjectSourceConfigsMock: vi.fn(), @@ -65,6 +68,7 @@ vi.mock("@/lib/api", () => ({ getProjectBlueskyCredentials: getProjectBlueskyCredentialsMock, getProjectIngestionRuns: getProjectIngestionRunsMock, getProjectIntakeAllowlist: getProjectIntakeAllowlistMock, + getProjectMastodonCredentials: getProjectMastodonCredentialsMock, getProjectNewsletterIntakes: getProjectNewsletterIntakesMock, getProjects: getProjectsMock, getProjectSourceConfigs: getProjectSourceConfigsMock, @@ -183,6 +187,24 @@ function createBlueskyCredentials( } } +function createMastodonCredentials( + overrides: Partial = {}, +): MastodonCredentials { + return { + id: 8, + project: 1, + instance_url: "https://hachyderm.io", + account_acct: "project@hachyderm.io", + is_active: true, + has_stored_credential: true, + last_verified_at: "2026-04-29T10:00:00Z", + last_error: "", + created_at: "2026-04-29T09:00:00Z", + updated_at: "2026-04-29T10:00:00Z", + ...overrides, + } +} + async function loadSourcesPageModule() { return import("../page") } @@ -245,14 +267,16 @@ describe("SourcesPage", () => { getProjectSourceConfigsMock.mockReset() getProjectIngestionRunsMock.mockReset() getProjectIntakeAllowlistMock.mockReset() + getProjectMastodonCredentialsMock.mockReset() getProjectNewsletterIntakesMock.mockReset() selectProjectMock.mockReset() - getProjectBlueskyCredentialsMock.mockResolvedValue([]) + getProjectBlueskyCredentialsMock.mockResolvedValue([]) getProjectsMock.mockResolvedValue([defaultProject]) getProjectSourceConfigsMock.mockResolvedValue([]) getProjectIngestionRunsMock.mockResolvedValue([]) getProjectIntakeAllowlistMock.mockResolvedValue([]) + getProjectMastodonCredentialsMock.mockResolvedValue([]) getProjectNewsletterIntakesMock.mockResolvedValue([]) selectProjectMock.mockImplementation((projects: Project[]) => { return projects[0] ?? null @@ -276,6 +300,7 @@ describe("SourcesPage", () => { expect(getProjectIngestionRunsMock).not.toHaveBeenCalled() expect(getProjectBlueskyCredentialsMock).not.toHaveBeenCalled() expect(getProjectIntakeAllowlistMock).not.toHaveBeenCalled() + expect(getProjectMastodonCredentialsMock).not.toHaveBeenCalled() expect(getProjectNewsletterIntakesMock).not.toHaveBeenCalled() }) @@ -315,6 +340,7 @@ describe("SourcesPage", () => { expect(getProjectIngestionRunsMock).toHaveBeenCalledWith(1) expect(getProjectBlueskyCredentialsMock).toHaveBeenCalledWith(1) expect(getProjectIntakeAllowlistMock).toHaveBeenCalledWith(1) + expect(getProjectMastodonCredentialsMock).toHaveBeenCalledWith(1) expect(getProjectNewsletterIntakesMock).toHaveBeenCalledWith(1) }) @@ -423,6 +449,28 @@ describe("SourcesPage", () => { ).toBeInTheDocument() }) + it("renders Mastodon verification controls from stored credentials", async () => { + const selectedProject = createProject({ id: 4 }) + + getProjectsMock.mockResolvedValue([selectedProject]) + selectProjectMock.mockReturnValue(selectedProject) + getProjectMastodonCredentialsMock.mockResolvedValue([ + createMastodonCredentials({ project: 4, account_acct: "alice@hachyderm.io" }), + ]) + + await renderSourcesPage({ project: "4" }) + + expect(screen.getByText("alice@hachyderm.io")).toBeInTheDocument() + expect( + screen.getByText( + "Save an optional per-instance access token for higher rate limits, then verify it without leaving the editor dashboard.", + ), + ).toBeInTheDocument() + expect( + screen.getByRole("button", { name: "Verify Mastodon credentials" }), + ).toBeEnabled() + }) + it("renders source cards with badge tones and the latest run summary", async () => { const selectedProject = createProject({ id: 3 }) getProjectsMock.mockResolvedValue([selectedProject]) @@ -474,7 +522,7 @@ describe("SourcesPage", () => { expect(screen.getByText("Rate limited")).toBeInTheDocument() const badges = screen.getAllByTestId("status-badge") - expect(badges).toHaveLength(4) + expect(badges).toHaveLength(5) expect( badges.some( (badge) => @@ -501,4 +549,12 @@ describe("SourcesPage", () => { expect(screen.getByText("Latest run: none")).toBeInTheDocument() expect(screen.getByText("No recent error")).toBeInTheDocument() }) + + it("includes Mastodon in the source creation options", async () => { + await renderSourcesPage({ project: "1" }) + + expect( + screen.getByRole("option", { name: "Mastodon" }), + ).toBeInTheDocument() + }) }) diff --git a/frontend/src/app/admin/sources/page.tsx b/frontend/src/app/admin/sources/page.tsx index 365823cb..ab9c1748 100644 --- a/frontend/src/app/admin/sources/page.tsx +++ b/frontend/src/app/admin/sources/page.tsx @@ -5,11 +5,17 @@ import { getProjectBlueskyCredentials, getProjectIngestionRuns, getProjectIntakeAllowlist, + getProjectMastodonCredentials, getProjectNewsletterIntakes, getProjects, getProjectSourceConfigs, } from "@/lib/api" -import type { BlueskyCredentials, NewsletterIntake, Project } from "@/lib/types" +import type { + BlueskyCredentials, + MastodonCredentials, + NewsletterIntake, + Project, +} from "@/lib/types" import { formatDate, getErrorMessage, @@ -60,6 +66,30 @@ export function deriveBlueskyVerificationState( return { label: "needs verification", tone: "warning" } } +/** + * Derive the current Mastodon verification badge state for stored credentials. + * + * @param credentials - Current stored Mastodon credentials, if any. + * @returns A badge label and semantic tone describing the stored credential state. + */ +export function deriveMastodonVerificationState( + credentials: MastodonCredentials | null, +): BlueskyVerificationState { + if (!credentials) { + return { label: "not configured", tone: "neutral" } + } + + if (credentials.last_error) { + return { label: "verification failed", tone: "negative" } + } + + if (credentials.last_verified_at) { + return { label: "verified", tone: "positive" } + } + + return { label: "needs verification", tone: "warning" } +} + type SourcesPageProps = { searchParams: Promise> } @@ -173,12 +203,14 @@ export default async function SourcesPage({ searchParams }: SourcesPageProps) { intakeAllowlist, newsletterIntakes, blueskyCredentials, + mastodonCredentials, ] = await Promise.all([ getProjectSourceConfigs(selectedProject.id), getProjectIngestionRuns(selectedProject.id), getProjectIntakeAllowlist(selectedProject.id), getProjectNewsletterIntakes(selectedProject.id), getProjectBlueskyCredentials(selectedProject.id), + getProjectMastodonCredentials(selectedProject.id), ]) const latestRunByPlugin = buildLatestRunByPlugin(ingestionRuns) const blueskyVerificationState = deriveBlueskyVerificationState(selectedProject) @@ -205,6 +237,11 @@ export default async function SourcesPage({ searchParams }: SourcesPageProps) { null const currentBlueskyCredentials: BlueskyCredentials | null = blueskyCredentials[0] ?? null + const currentMastodonCredentials: MastodonCredentials | null = + mastodonCredentials[0] ?? null + const mastodonVerificationState = deriveMastodonVerificationState( + currentMastodonCredentials, + ) const errorMessage = getErrorMessage(resolvedSearchParams) const successMessage = getSuccessMessage(resolvedSearchParams) @@ -212,7 +249,7 @@ export default async function SourcesPage({ searchParams }: SourcesPageProps) { return ( @@ -654,6 +691,135 @@ export default async function SourcesPage({ searchParams }: SourcesPageProps) { +
+
+
+

Mastodon

+

+ Credential verification +

+

+ Save an optional per-instance access token for higher rate limits, then + verify it without leaving the editor dashboard. +

+
+ + {mastodonVerificationState.label} + +
+ +
+
+

+ Stored credentials +

+

+ {currentMastodonCredentials + ? currentMastodonCredentials.account_acct || currentMastodonCredentials.instance_url + : "No Mastodon credentials are configured for this project yet."} +

+

+ {currentMastodonCredentials?.last_verified_at + ? `Last verified ${formatDate(currentMastodonCredentials.last_verified_at)}` + : "Run verification after saving credentials to confirm the token."} +

+ {currentMastodonCredentials?.last_error ? ( +

+ {currentMastodonCredentials.last_error} +

+ ) : null} +
+
+

+ Save credentials +

+
+ + + + + + + +
+

+ Use {"{\"instance_url\": \"https://hachyderm.io\", \"hashtag\": \"platformengineering\"}"} for a hashtag timeline, {"{\"account_acct\": \"alice@hachyderm.io\"}"} for an account, or {"{\"list_id\": 42}"} for a list. +

+
+
+ +
+ + +
+
+

Add source

RSS +

- Bluesky configs accept either an actor handle or a feed URI. RSS and Reddit - continue to use the existing backend JSON shapes. + Bluesky configs accept either an actor handle or a feed URI. Mastodon + configs accept an instance URL plus one of hashtag, account_acct, or list_id. RSS and Reddit continue to use the existing backend JSON shapes.