diff --git a/flake.nix b/flake.nix index 31f283293971..8daf7b017b97 100644 --- a/flake.nix +++ b/flake.nix @@ -104,6 +104,7 @@ libxml2 libxslt sqlite + gcc # Native dependencies for unit tests (SyTest also requires OpenSSL). openssl @@ -152,7 +153,7 @@ # Postgres is needed to run Synapse with postgres support and # to run certain unit tests that require postgres. - services.postgres.enable = true; + services.postgres.enable = false; # On the first invocation of `devenv up`, create a database for # Synapse to store data in. @@ -168,7 +169,7 @@ ''; # Redis is needed in order to run Synapse in worker mode. - services.redis.enable = true; + services.redis.enable = false; # Configure and start Synapse. Before starting Synapse, this shell code: # * generates a default homeserver.yaml config file if one does not exist, and @@ -178,22 +179,6 @@ process.before = '' python -m synapse.app.homeserver -c homeserver.yaml --generate-config --server-name=synapse.dev --report-stats=no mkdir -p homeserver-config-overrides.d - cat > homeserver-config-overrides.d/database.yaml << EOF - ## Do not edit this file. This file is generated by flake.nix - database: - name: psycopg2 - args: - user: synapse_user - database: synapse - host: $PGHOST - cp_min: 5 - cp_max: 10 - EOF - cat > homeserver-config-overrides.d/redis.yaml << EOF - ## Do not edit this file. This file is generated by flake.nix - redis: - enabled: true - EOF ''; # Start synapse when `devenv up` is run. processes.synapse.exec = "poetry run python -m synapse.app.homeserver -c homeserver.yaml -c homeserver-config-overrides.d"; diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 7fd46901f71e..e3b18b53e98b 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -78,6 +78,7 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.max_upload_size = hs.config.media.max_upload_size self.max_image_pixels = hs.config.media.max_image_pixels + self.unused_expiration_time = hs.config.media.unused_expiration_time Thumbnailer.set_limits(self.max_image_pixels) @@ -183,6 +184,29 @@ def mark_recently_accessed(self, server_name: Optional[str], media_id: str) -> N else: self.recently_accessed_locals.add(media_id) + @trace + async def create_media_id(self, auth_user: UserID) -> Tuple[str, int]: + """Create and store a media ID for a local user and return the MXC URI and its + expiration. + + Args: + auth_user: The user_id of the uploader + + Returns: + A tuple containing the MXC URI of the stored content and the timestamp at + which the MXC URI expires. + """ + media_id = random_string(24) + now = self.clock.time_msec() + unused_expires_at = now + self.unused_expiration_time + await self.store.store_local_media_id( + media_id=media_id, + time_now_ms=now, + user_id=auth_user, + unused_expires_at=unused_expires_at, + ) + return f"mxc://{self.server_name}/{media_id}", unused_expires_at + @trace async def create_content( self, diff --git a/synapse/rest/media/create_resource.py b/synapse/rest/media/create_resource.py new file mode 100644 index 000000000000..cbb36e5fb710 --- /dev/null +++ b/synapse/rest/media/create_resource.py @@ -0,0 +1,88 @@ +# Copyright 2023 Beeper Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import re +from typing import TYPE_CHECKING + +from synapse.api.errors import LimitExceededError +from synapse.api.ratelimiting import Ratelimiter +from synapse.http.server import respond_with_json +from synapse.http.servlet import RestServlet +from synapse.http.site import SynapseRequest + +if TYPE_CHECKING: + from synapse.media.media_repository import MediaRepository + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class CreateResource(RestServlet): + PATTERNS = [re.compile("/_matrix/media/v1/create")] + isLeaf = True + + def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): + super().__init__() + + self.media_repo = media_repo + self.clock = hs.get_clock() + self.auth = hs.get_auth() + + # A rate limiter for creating new media IDs. + self._create_media_rate_limiter = Ratelimiter( + store=hs.get_datastores().main, + clock=self.clock, + cfg=hs.config.ratelimiting.rc_media_create, + ) + + async def _async_render_OPTIONS(self, request: SynapseRequest) -> None: + respond_with_json(request, 200, {}, send_cors=True) + + async def _async_render_POST(self, request: SynapseRequest) -> None: + requester = await self.auth.get_user_by_req(request) + + # If the create media requests for the user are over the limit, drop them. + await self._create_media_rate_limiter.ratelimit(requester) + + ( + reached_pending_limit, + first_expiration_ts, + ) = await self.media_repo.reached_pending_media_limit( + requester.user, self.max_pending_media_uploads + ) + if reached_pending_limit: + raise LimitExceededError( + limiter_name="max_pending_media_uploads", + retry_after_ms=first_expiration_ts - self.clock.time_msec(), + ) + + content_uri, unused_expires_at = await self.media_repo.create_media_id( + requester.user + ) + + logger.info( + "Created Media URI %r that if unused will expire at %d", + content_uri, + unused_expires_at, + ) + respond_with_json( + request, + 200, + { + "content_uri": content_uri, + "unused_expires_at": unused_expires_at, + }, + send_cors=True, + ) diff --git a/synapse/rest/media/media_repository_resource.py b/synapse/rest/media/media_repository_resource.py index 2089bb10296c..e92b0716ca1a 100644 --- a/synapse/rest/media/media_repository_resource.py +++ b/synapse/rest/media/media_repository_resource.py @@ -18,6 +18,7 @@ from synapse.http.server import HttpServer, JsonResource from .config_resource import MediaConfigResource +from .create_resource import CreateResource from .download_resource import DownloadResource from .preview_url_resource import PreviewUrlResource from .thumbnail_resource import ThumbnailResource @@ -91,7 +92,7 @@ def register_servlets(http_server: HttpServer, hs: "HomeServer") -> None: # Note that many of these should not exist as v1 endpoints, but empirically # a lot of traffic still goes to them. - + CreateResource(hs, media_repo).register(http_server) UploadResource(hs, media_repo).register(http_server) DownloadResource(hs, media_repo).register(http_server) ThumbnailResource(hs, media_repo, media_repo.media_storage).register( diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index 2e6b176bd2c2..8ad71db81b01 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -330,6 +330,25 @@ def _get_local_media_ids_txn(txn: LoggingTransaction) -> List[str]: "get_local_media_ids", _get_local_media_ids_txn ) + @trace + async def store_local_media_id( + self, + media_id: str, + time_now_ms: int, + user_id: UserID, + unused_expires_at: int, + ) -> None: + await self.db_pool.simple_insert( + "local_media_repository", + { + "media_id": media_id, + "created_ts": time_now_ms, + "user_id": user_id.to_string(), + "unused_expires_at": unused_expires_at, + }, + desc="store_local_media_id", + ) + @trace async def store_local_media( self, diff --git a/synapse/storage/schema/main/delta/80/02_add_unused_expires_at_for_media.sql b/synapse/storage/schema/main/delta/80/02_add_unused_expires_at_for_media.sql new file mode 100644 index 000000000000..ed464820cece --- /dev/null +++ b/synapse/storage/schema/main/delta/80/02_add_unused_expires_at_for_media.sql @@ -0,0 +1,22 @@ +/* Copyright 2023 Beeper Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Add new colums to the `local_media_repository` to keep track of when the +-- media ID must be used by. This is to support async uploads (see MSC2246). + +ALTER TABLE local_media_repository + ADD COLUMN unused_expires_at BIGINT DEFAULT NULL; + +CREATE INDEX CONCURRENTLY ON local_media_repository (unused_expires_at);