View
97 synapse/rest/client/v1/login.py
@@ -19,6 +19,7 @@
from synapse.types import UserID
from synapse.http.server import finish_request
from synapse.http.servlet import parse_json_object_from_request
+from synapse.util.msisdn import phone_number_to_msisdn
from .base import ClientV1RestServlet, client_path_patterns
@@ -33,10 +34,55 @@
import xml.etree.ElementTree as ET
+from twisted.web.client import PartialDownloadError
+
logger = logging.getLogger(__name__)
+def login_submission_legacy_convert(submission):
+ """
+ If the input login submission is an old style object
+ (ie. with top-level user / medium / address) convert it
+ to a typed object.
+ """
+ if "user" in submission:
+ submission["identifier"] = {
+ "type": "m.id.user",
+ "user": submission["user"],
+ }
+ del submission["user"]
+
+ if "medium" in submission and "address" in submission:
+ submission["identifier"] = {
+ "type": "m.id.thirdparty",
+ "medium": submission["medium"],
+ "address": submission["address"],
+ }
+ del submission["medium"]
+ del submission["address"]
+
+
+def login_id_thirdparty_from_phone(identifier):
+ """
+ Convert a phone login identifier type to a generic threepid identifier
+ Args:
+ identifier(dict): Login identifier dict of type 'm.id.phone'
+
+ Returns: Login identifier dict of type 'm.id.threepid'
+ """
+ if "country" not in identifier or "number" not in identifier:
+ raise SynapseError(400, "Invalid phone-type identifier")
+
+ msisdn = phone_number_to_msisdn(identifier["country"], identifier["number"])
+
+ return {
+ "type": "m.id.thirdparty",
+ "medium": "msisdn",
+ "address": msisdn,
+ }
+
+
class LoginRestServlet(ClientV1RestServlet):
PATTERNS = client_path_patterns("/login$")
PASS_TYPE = "m.login.password"
@@ -117,20 +163,52 @@ def on_POST(self, request):
@defer.inlineCallbacks
def do_password_login(self, login_submission):
- if 'medium' in login_submission and 'address' in login_submission:
- address = login_submission['address']
- if login_submission['medium'] == 'email':
+ if "password" not in login_submission:
+ raise SynapseError(400, "Missing parameter: password")
+
+ login_submission_legacy_convert(login_submission)
+
+ if "identifier" not in login_submission:
+ raise SynapseError(400, "Missing param: identifier")
+
+ identifier = login_submission["identifier"]
+ if "type" not in identifier:
+ raise SynapseError(400, "Login identifier has no type")
+
+ # convert phone type identifiers to generic threepids
+ if identifier["type"] == "m.id.phone":
+ identifier = login_id_thirdparty_from_phone(identifier)
+
+ # convert threepid identifiers to user IDs
+ if identifier["type"] == "m.id.thirdparty":
+ if 'medium' not in identifier or 'address' not in identifier:
+ raise SynapseError(400, "Invalid thirdparty identifier")
+
+ address = identifier['address']
+ if identifier['medium'] == 'email':
# For emails, transform the address to lowercase.
# We store all email addreses as lowercase in the DB.
# (See add_threepid in synapse/handlers/auth.py)
address = address.lower()
user_id = yield self.hs.get_datastore().get_user_id_by_threepid(
- login_submission['medium'], address
+ identifier['medium'], address
)
if not user_id:
raise LoginError(403, "", errcode=Codes.FORBIDDEN)
- else:
- user_id = login_submission['user']
+
+ identifier = {
+ "type": "m.id.user",
+ "user": user_id,
+ }
+
+ # by this point, the identifier should be an m.id.user: if it's anything
+ # else, we haven't understood it.
+ if identifier["type"] != "m.id.user":
+ raise SynapseError(400, "Unknown login identifier type")
+ if "user" not in identifier:
+ raise SynapseError(400, "User identifier is missing 'user' key")
+
+ user_id = identifier["user"]
if not user_id.startswith('@'):
user_id = UserID.create(
@@ -341,7 +419,12 @@ def on_GET(self, request):
"ticket": request.args["ticket"],
"service": self.cas_service_url
}
- body = yield http_client.get_raw(uri, args)
+ try:
+ body = yield http_client.get_raw(uri, args)
+ except PartialDownloadError as pde:
+ # Twisted raises this error if the connection is closed,
+ # even if that's being used old-http style to signal end-of-data
+ body = pde.response
result = yield self.handle_cas_response(request, body, client_redirect_url)
defer.returnValue(result)
View
3 synapse/rest/client/v1/presence.py
@@ -19,6 +19,7 @@
from synapse.api.errors import SynapseError, AuthError
from synapse.types import UserID
+from synapse.handlers.presence import format_user_presence_state
from synapse.http.servlet import parse_json_object_from_request
from .base import ClientV1RestServlet, client_path_patterns
@@ -33,6 +34,7 @@ class PresenceStatusRestServlet(ClientV1RestServlet):
def __init__(self, hs):
super(PresenceStatusRestServlet, self).__init__(hs)
self.presence_handler = hs.get_presence_handler()
+ self.clock = hs.get_clock()
@defer.inlineCallbacks
def on_GET(self, request, user_id):
@@ -48,6 +50,7 @@ def on_GET(self, request, user_id):
raise AuthError(403, "You are not allowed to see their presence.")
state = yield self.presence_handler.get_state(target_user=user)
+ state = format_user_presence_state(state, self.clock.time_msec())
defer.returnValue((200, state))
View
3 synapse/rest/client/v1/room.py
@@ -748,8 +748,7 @@ def __init__(self, hs):
def on_GET(self, request):
requester = yield self.auth.get_user_by_req(request, allow_guest=True)
- rooms = yield self.store.get_rooms_for_user(requester.user.to_string())
- room_ids = set(r.room_id for r in rooms) # Ensure they're unique.
+ room_ids = yield self.store.get_rooms_for_user(requester.user.to_string())
defer.returnValue((200, {"joined_rooms": list(room_ids)}))
View
114 synapse/rest/client/v2_alpha/account.py
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2015, 2016 OpenMarket Ltd
+# Copyright 2017 Vector Creations Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -17,8 +18,11 @@
from synapse.api.constants import LoginType
from synapse.api.errors import LoginError, SynapseError, Codes
-from synapse.http.servlet import RestServlet, parse_json_object_from_request
+from synapse.http.servlet import (
+ RestServlet, parse_json_object_from_request, assert_params_in_request
+)
from synapse.util.async import run_on_reactor
+from synapse.util.msisdn import phone_number_to_msisdn
from ._base import client_v2_patterns
@@ -28,26 +32,21 @@
logger = logging.getLogger(__name__)
-class PasswordRequestTokenRestServlet(RestServlet):
+class EmailPasswordRequestTokenRestServlet(RestServlet):
PATTERNS = client_v2_patterns("/account/password/email/requestToken$")
def __init__(self, hs):
- super(PasswordRequestTokenRestServlet, self).__init__()
+ super(EmailPasswordRequestTokenRestServlet, self).__init__()
self.hs = hs
self.identity_handler = hs.get_handlers().identity_handler
@defer.inlineCallbacks
def on_POST(self, request):
body = parse_json_object_from_request(request)
- required = ['id_server', 'client_secret', 'email', 'send_attempt']
- absent = []
- for k in required:
- if k not in body:
- absent.append(k)
-
- if absent:
- raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM)
+ assert_params_in_request(body, [
+ 'id_server', 'client_secret', 'email', 'send_attempt'
+ ])
existingUid = yield self.hs.get_datastore().get_user_id_by_threepid(
'email', body['email']
@@ -60,6 +59,37 @@ def on_POST(self, request):
defer.returnValue((200, ret))
+class MsisdnPasswordRequestTokenRestServlet(RestServlet):
+ PATTERNS = client_v2_patterns("/account/password/msisdn/requestToken$")
+
+ def __init__(self, hs):
+ super(MsisdnPasswordRequestTokenRestServlet, self).__init__()
+ self.hs = hs
+ self.datastore = self.hs.get_datastore()
+ self.identity_handler = hs.get_handlers().identity_handler
+
+ @defer.inlineCallbacks
+ def on_POST(self, request):
+ body = parse_json_object_from_request(request)
+
+ assert_params_in_request(body, [
+ 'id_server', 'client_secret',
+ 'country', 'phone_number', 'send_attempt',
+ ])
+
+ msisdn = phone_number_to_msisdn(body['country'], body['phone_number'])
+
+ existingUid = yield self.datastore.get_user_id_by_threepid(
+ 'msisdn', msisdn
+ )
+
+ if existingUid is None:
+ raise SynapseError(400, "MSISDN not found", Codes.THREEPID_NOT_FOUND)
+
+ ret = yield self.identity_handler.requestMsisdnToken(**body)
+ defer.returnValue((200, ret))
+
+
class PasswordRestServlet(RestServlet):
PATTERNS = client_v2_patterns("/account/password$")
@@ -68,6 +98,7 @@ def __init__(self, hs):
self.hs = hs
self.auth = hs.get_auth()
self.auth_handler = hs.get_auth_handler()
+ self.datastore = self.hs.get_datastore()
@defer.inlineCallbacks
def on_POST(self, request):
@@ -77,7 +108,8 @@ def on_POST(self, request):
authed, result, params, _ = yield self.auth_handler.check_auth([
[LoginType.PASSWORD],
- [LoginType.EMAIL_IDENTITY]
+ [LoginType.EMAIL_IDENTITY],
+ [LoginType.MSISDN],
], body, self.hs.get_ip_from_request(request))
if not authed:
@@ -102,7 +134,7 @@ def on_POST(self, request):
# (See add_threepid in synapse/handlers/auth.py)
threepid['address'] = threepid['address'].lower()
# if using email, we must know about the email they're authing with!
- threepid_user_id = yield self.hs.get_datastore().get_user_id_by_threepid(
+ threepid_user_id = yield self.datastore.get_user_id_by_threepid(
threepid['medium'], threepid['address']
)
if not threepid_user_id:
@@ -169,13 +201,14 @@ def on_POST(self, request):
defer.returnValue((200, {}))
-class ThreepidRequestTokenRestServlet(RestServlet):
+class EmailThreepidRequestTokenRestServlet(RestServlet):
PATTERNS = client_v2_patterns("/account/3pid/email/requestToken$")
def __init__(self, hs):
self.hs = hs
- super(ThreepidRequestTokenRestServlet, self).__init__()
+ super(EmailThreepidRequestTokenRestServlet, self).__init__()
self.identity_handler = hs.get_handlers().identity_handler
+ self.datastore = self.hs.get_datastore()
@defer.inlineCallbacks
def on_POST(self, request):
@@ -190,7 +223,7 @@ def on_POST(self, request):
if absent:
raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM)
- existingUid = yield self.hs.get_datastore().get_user_id_by_threepid(
+ existingUid = yield self.datastore.get_user_id_by_threepid(
'email', body['email']
)
@@ -201,6 +234,44 @@ def on_POST(self, request):
defer.returnValue((200, ret))
+class MsisdnThreepidRequestTokenRestServlet(RestServlet):
+ PATTERNS = client_v2_patterns("/account/3pid/msisdn/requestToken$")
+
+ def __init__(self, hs):
+ self.hs = hs
+ super(MsisdnThreepidRequestTokenRestServlet, self).__init__()
+ self.identity_handler = hs.get_handlers().identity_handler
+ self.datastore = self.hs.get_datastore()
+
+ @defer.inlineCallbacks
+ def on_POST(self, request):
+ body = parse_json_object_from_request(request)
+
+ required = [
+ 'id_server', 'client_secret',
+ 'country', 'phone_number', 'send_attempt',
+ ]
+ absent = []
+ for k in required:
+ if k not in body:
+ absent.append(k)
+
+ if absent:
+ raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM)
+
+ msisdn = phone_number_to_msisdn(body['country'], body['phone_number'])
+
+ existingUid = yield self.datastore.get_user_id_by_threepid(
+ 'msisdn', msisdn
+ )
+
+ if existingUid is not None:
+ raise SynapseError(400, "MSISDN is already in use", Codes.THREEPID_IN_USE)
+
+ ret = yield self.identity_handler.requestEmailToken(**body)
+ defer.returnValue((200, ret))
+
+
class ThreepidRestServlet(RestServlet):
PATTERNS = client_v2_patterns("/account/3pid$")
@@ -210,14 +281,15 @@ def __init__(self, hs):
self.identity_handler = hs.get_handlers().identity_handler
self.auth = hs.get_auth()
self.auth_handler = hs.get_auth_handler()
+ self.datastore = self.hs.get_datastore()
@defer.inlineCallbacks
def on_GET(self, request):
yield run_on_reactor()
requester = yield self.auth.get_user_by_req(request)
- threepids = yield self.hs.get_datastore().user_get_threepids(
+ threepids = yield self.datastore.user_get_threepids(
requester.user.to_string()
)
@@ -258,7 +330,7 @@ def on_POST(self, request):
if 'bind' in body and body['bind']:
logger.debug(
- "Binding emails %s to %s",
+ "Binding threepid %s to %s",
threepid, user_id
)
yield self.identity_handler.bind_threepid(
@@ -302,9 +374,11 @@ def on_POST(self, request):
def register_servlets(hs, http_server):
- PasswordRequestTokenRestServlet(hs).register(http_server)
+ EmailPasswordRequestTokenRestServlet(hs).register(http_server)
+ MsisdnPasswordRequestTokenRestServlet(hs).register(http_server)
PasswordRestServlet(hs).register(http_server)
DeactivateAccountRestServlet(hs).register(http_server)
- ThreepidRequestTokenRestServlet(hs).register(http_server)
+ EmailThreepidRequestTokenRestServlet(hs).register(http_server)
+ MsisdnThreepidRequestTokenRestServlet(hs).register(http_server)
ThreepidRestServlet(hs).register(http_server)
ThreepidDeleteRestServlet(hs).register(http_server)
View
47 synapse/rest/client/v2_alpha/devices.py
@@ -46,6 +46,52 @@ def on_GET(self, request):
defer.returnValue((200, {"devices": devices}))
+class DeleteDevicesRestServlet(servlet.RestServlet):
+ """
+ API for bulk deletion of devices. Accepts a JSON object with a devices
+ key which lists the device_ids to delete. Requires user interactive auth.
+ """
+ PATTERNS = client_v2_patterns("/delete_devices", releases=[], v2_alpha=False)
+
+ def __init__(self, hs):
+ super(DeleteDevicesRestServlet, self).__init__()
+ self.hs = hs
+ self.auth = hs.get_auth()
+ self.device_handler = hs.get_device_handler()
+ self.auth_handler = hs.get_auth_handler()
+
+ @defer.inlineCallbacks
+ def on_POST(self, request):
+ try:
+ body = servlet.parse_json_object_from_request(request)
+ except errors.SynapseError as e:
+ if e.errcode == errors.Codes.NOT_JSON:
+ # deal with older clients which didn't pass a J*DELETESON dict
+ # the same as those that pass an empty dict
+ body = {}
+ else:
+ raise e
+
+ if 'devices' not in body:
+ raise errors.SynapseError(
+ 400, "No devices supplied", errcode=errors.Codes.MISSING_PARAM
+ )
+
+ authed, result, params, _ = yield self.auth_handler.check_auth([
+ [constants.LoginType.PASSWORD],
+ ], body, self.hs.get_ip_from_request(request))
+
+ if not authed:
+ defer.returnValue((401, result))
+
+ requester = yield self.auth.get_user_by_req(request)
+ yield self.device_handler.delete_devices(
+ requester.user.to_string(),
+ body['devices'],
+ )
+ defer.returnValue((200, {}))
+
+
class DeviceRestServlet(servlet.RestServlet):
PATTERNS = client_v2_patterns("/devices/(?P<device_id>[^/]*)$",
releases=[], v2_alpha=False)
@@ -111,5 +157,6 @@ def on_PUT(self, request, device_id):
def register_servlets(hs, http_server):
+ DeleteDevicesRestServlet(hs).register(http_server)
DevicesRestServlet(hs).register(http_server)
DeviceRestServlet(hs).register(http_server)
View
137 synapse/rest/client/v2_alpha/register.py
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2015 - 2016 OpenMarket Ltd
+# Copyright 2017 Vector Creations Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -19,7 +20,10 @@
from synapse.api.auth import get_access_token_from_request, has_access_token
from synapse.api.constants import LoginType
from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError
-from synapse.http.servlet import RestServlet, parse_json_object_from_request
+from synapse.http.servlet import (
+ RestServlet, parse_json_object_from_request, assert_params_in_request
+)
+from synapse.util.msisdn import phone_number_to_msisdn
from ._base import client_v2_patterns
@@ -43,30 +47,25 @@ def compare_digest(a, b):
logger = logging.getLogger(__name__)
-class RegisterRequestTokenRestServlet(RestServlet):
+class EmailRegisterRequestTokenRestServlet(RestServlet):
PATTERNS = client_v2_patterns("/register/email/requestToken$")
def __init__(self, hs):
"""
Args:
hs (synapse.server.HomeServer): server
"""
- super(RegisterRequestTokenRestServlet, self).__init__()
+ super(EmailRegisterRequestTokenRestServlet, self).__init__()
self.hs = hs
self.identity_handler = hs.get_handlers().identity_handler
@defer.inlineCallbacks
def on_POST(self, request):
body = parse_json_object_from_request(request)
- required = ['id_server', 'client_secret', 'email', 'send_attempt']
- absent = []
- for k in required:
- if k not in body:
- absent.append(k)
-
- if len(absent) > 0:
- raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM)
+ assert_params_in_request(body, [
+ 'id_server', 'client_secret', 'email', 'send_attempt'
+ ])
existingUid = yield self.hs.get_datastore().get_user_id_by_threepid(
'email', body['email']
@@ -79,6 +78,43 @@ def on_POST(self, request):
defer.returnValue((200, ret))
+class MsisdnRegisterRequestTokenRestServlet(RestServlet):
+ PATTERNS = client_v2_patterns("/register/msisdn/requestToken$")
+
+ def __init__(self, hs):
+ """
+ Args:
+ hs (synapse.server.HomeServer): server
+ """
+ super(MsisdnRegisterRequestTokenRestServlet, self).__init__()
+ self.hs = hs
+ self.identity_handler = hs.get_handlers().identity_handler
+
+ @defer.inlineCallbacks
+ def on_POST(self, request):
+ body = parse_json_object_from_request(request)
+
+ assert_params_in_request(body, [
+ 'id_server', 'client_secret',
+ 'country', 'phone_number',
+ 'send_attempt',
+ ])
+
+ msisdn = phone_number_to_msisdn(body['country'], body['phone_number'])
+
+ existingUid = yield self.hs.get_datastore().get_user_id_by_threepid(
+ 'msisdn', msisdn
+ )
+
+ if existingUid is not None:
+ raise SynapseError(
+ 400, "Phone number is already in use", Codes.THREEPID_IN_USE
+ )
+
+ ret = yield self.identity_handler.requestMsisdnToken(**body)
+ defer.returnValue((200, ret))
+
+
class RegisterRestServlet(RestServlet):
PATTERNS = client_v2_patterns("/register$")
@@ -200,16 +236,37 @@ def on_POST(self, request):
assigned_user_id=registered_user_id,
)
+ # Only give msisdn flows if the x_show_msisdn flag is given:
+ # this is a hack to work around the fact that clients were shipped
+ # that use fallback registration if they see any flows that they don't
+ # recognise, which means we break registration for these clients if we
+ # advertise msisdn flows. Once usage of Riot iOS <=0.3.9 and Riot
+ # Android <=0.6.9 have fallen below an acceptable threshold, this
+ # parameter should go away and we should always advertise msisdn flows.
+ show_msisdn = False
+ if 'x_show_msisdn' in body and body['x_show_msisdn']:
+ show_msisdn = True
+
if self.hs.config.enable_registration_captcha:
flows = [
[LoginType.RECAPTCHA],
- [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA]
+ [LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA],
]
+ if show_msisdn:
+ flows.extend([
+ [LoginType.MSISDN, LoginType.RECAPTCHA],
+ [LoginType.MSISDN, LoginType.EMAIL_IDENTITY, LoginType.RECAPTCHA],
+ ])
else:
flows = [
[LoginType.DUMMY],
- [LoginType.EMAIL_IDENTITY]
+ [LoginType.EMAIL_IDENTITY],
]
+ if show_msisdn:
+ flows.extend([
+ [LoginType.MSISDN],
+ [LoginType.MSISDN, LoginType.EMAIL_IDENTITY],
+ ])
authed, auth_result, params, session_id = yield self.auth_handler.check_auth(
flows, body, self.hs.get_ip_from_request(request)
@@ -224,8 +281,9 @@ def on_POST(self, request):
"Already registered user ID %r for this session",
registered_user_id
)
- # don't re-register the email address
+ # don't re-register the threepids
add_email = False
+ add_msisdn = False
else:
# NB: This may be from the auth handler and NOT from the POST
if 'password' not in params:
@@ -250,6 +308,7 @@ def on_POST(self, request):
)
add_email = True
+ add_msisdn = True
return_dict = yield self._create_registration_details(
registered_user_id, params
@@ -262,6 +321,13 @@ def on_POST(self, request):
params.get("bind_email")
)
+ if add_msisdn and auth_result and LoginType.MSISDN in auth_result:
+ threepid = auth_result[LoginType.MSISDN]
+ yield self._register_msisdn_threepid(
+ registered_user_id, threepid, return_dict["access_token"],
+ params.get("bind_msisdn")
+ )
+
defer.returnValue((200, return_dict))
def on_OPTIONS(self, _):
@@ -323,8 +389,9 @@ def _register_email_threepid(self, user_id, threepid, token, bind_email):
"""
reqd = ('medium', 'address', 'validated_at')
if any(x not in threepid for x in reqd):
+ # This will only happen if the ID server returns a malformed response
logger.info("Can't add incomplete 3pid")
- defer.returnValue()
+ return
yield self.auth_handler.add_threepid(
user_id,
@@ -372,6 +439,43 @@ def _register_email_threepid(self, user_id, threepid, token, bind_email):
logger.info("bind_email not specified: not binding email")
@defer.inlineCallbacks
+ def _register_msisdn_threepid(self, user_id, threepid, token, bind_msisdn):
+ """Add a phone number as a 3pid identifier
+
+ Also optionally binds msisdn to the given user_id on the identity server
+
+ Args:
+ user_id (str): id of user
+ threepid (object): m.login.msisdn auth response
+ token (str): access_token for the user
+ bind_email (bool): true if the client requested the email to be
+ bound at the identity server
+ Returns:
+ defer.Deferred:
+ """
+ reqd = ('medium', 'address', 'validated_at')
+ if any(x not in threepid for x in reqd):
+ # This will only happen if the ID server returns a malformed response
+ logger.info("Can't add incomplete 3pid")
+ defer.returnValue()
+
+ yield self.auth_handler.add_threepid(
+ user_id,
+ threepid['medium'],
+ threepid['address'],
+ threepid['validated_at'],
+ )
+
+ if bind_msisdn:
+ logger.info("bind_msisdn specified: binding")
+ logger.debug("Binding msisdn %s to %s", threepid, user_id)
+ yield self.identity_handler.bind_threepid(
+ threepid['threepid_creds'], user_id
+ )
+ else:
+ logger.info("bind_msisdn not specified: not binding msisdn")
+
+ @defer.inlineCallbacks
def _create_registration_details(self, user_id, params):
"""Complete registration of newly-registered user
@@ -449,5 +553,6 @@ def _do_guest_registration(self, params):
def register_servlets(hs, http_server):
- RegisterRequestTokenRestServlet(hs).register(http_server)
+ EmailRegisterRequestTokenRestServlet(hs).register(http_server)
+ MsisdnRegisterRequestTokenRestServlet(hs).register(http_server)
RegisterRestServlet(hs).register(http_server)
View
20 synapse/rest/client/v2_alpha/sync.py
@@ -18,6 +18,7 @@
from synapse.http.servlet import (
RestServlet, parse_string, parse_integer, parse_boolean
)
+from synapse.handlers.presence import format_user_presence_state
from synapse.handlers.sync import SyncConfig
from synapse.types import StreamToken
from synapse.events.utils import (
@@ -28,7 +29,6 @@
from synapse.api.constants import PresenceState
from ._base import client_v2_patterns
-import copy
import itertools
import logging
@@ -194,12 +194,18 @@ def on_GET(self, request):
defer.returnValue((200, response_content))
def encode_presence(self, events, time_now):
- formatted = []
- for event in events:
- event = copy.deepcopy(event)
- event['sender'] = event['content'].pop('user_id')
- formatted.append(event)
- return {"events": formatted}
+ return {
+ "events": [
+ {
+ "type": "m.presence",
+ "sender": event.user_id,
+ "content": format_user_presence_state(
+ event, time_now, include_user_id=False
+ ),
+ }
+ for event in events
+ ]
+ }
def encode_joined(self, rooms, time_now, token_id, event_fields):
"""
View
12 synapse/rest/media/v1/download_resource.py
@@ -12,6 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+import synapse.http.servlet
from ._base import parse_media_id, respond_with_file, respond_404
from twisted.web.resource import Resource
@@ -81,6 +82,17 @@ def _respond_local_file(self, request, media_id, name):
@defer.inlineCallbacks
def _respond_remote_file(self, request, server_name, media_id, name):
+ # don't forward requests for remote media if allow_remote is false
+ allow_remote = synapse.http.servlet.parse_boolean(
+ request, "allow_remote", default=True)
+ if not allow_remote:
+ logger.info(
+ "Rejecting request for remote media %s/%s due to allow_remote",
+ server_name, media_id,
+ )
+ respond_404(request)
+ return
+
media_info = yield self.media_repo.get_remote_media(server_name, media_id)
media_type = media_info["media_type"]
View
44 synapse/rest/media/v1/media_repository.py
@@ -13,22 +13,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from twisted.internet import defer, threads
+import twisted.internet.error
+import twisted.web.http
+from twisted.web.resource import Resource
+
from .upload_resource import UploadResource
from .download_resource import DownloadResource
from .thumbnail_resource import ThumbnailResource
from .identicon_resource import IdenticonResource
from .preview_url_resource import PreviewUrlResource
from .filepath import MediaFilePaths
-
-from twisted.web.resource import Resource
-
from .thumbnailer import Thumbnailer
from synapse.http.matrixfederationclient import MatrixFederationHttpClient
from synapse.util.stringutils import random_string
-from synapse.api.errors import SynapseError
-
-from twisted.internet import defer, threads
+from synapse.api.errors import SynapseError, HttpResponseException, \
+ NotFoundError
from synapse.util.async import Linearizer
from synapse.util.stringutils import is_ascii
@@ -157,11 +158,34 @@ def _download_remote_file(self, server_name, media_id):
try:
length, headers = yield self.client.get_file(
server_name, request_path, output_stream=f,
- max_size=self.max_upload_size,
+ max_size=self.max_upload_size, args={
+ # tell the remote server to 404 if it doesn't
+ # recognise the server_name, to make sure we don't
+ # end up with a routing loop.
+ "allow_remote": "false",
+ }
)
- except Exception as e:
- logger.warn("Failed to fetch remoted media %r", e)
- raise SynapseError(502, "Failed to fetch remoted media")
+ except twisted.internet.error.DNSLookupError as e:
+ logger.warn("HTTP error fetching remote media %s/%s: %r",
+ server_name, media_id, e)
+ raise NotFoundError()
+
+ except HttpResponseException as e:
+ logger.warn("HTTP error fetching remote media %s/%s: %s",
+ server_name, media_id, e.response)
+ if e.code == twisted.web.http.NOT_FOUND:
+ raise SynapseError.from_http_response_exception(e)
+ raise SynapseError(502, "Failed to fetch remote media")
+
+ except SynapseError:
+ logger.exception("Failed to fetch remote media %s/%s",
+ server_name, media_id)
+ raise
+
+ except Exception:
+ logger.exception("Failed to fetch remote media %s/%s",
+ server_name, media_id)
+ raise SynapseError(502, "Failed to fetch remote media")
media_type = headers["Content-Type"][0]
time_now_ms = self.clock.time_msec()
View
11 synapse/state.py
@@ -177,17 +177,12 @@ def get_current_user_in_room(self, room_id, latest_event_ids=None):
@defer.inlineCallbacks
def compute_event_context(self, event, old_state=None):
- """ Fills out the context with the `current state` of the graph. The
- `current state` here is defined to be the state of the event graph
- just before the event - i.e. it never includes `event`
-
- If `event` has `auth_events` then this will also fill out the
- `auth_events` field on `context` from the `current_state`.
+ """Build an EventContext structure for the event.
Args:
- event (EventBase)
+ event (synapse.events.EventBase):
Returns:
- an EventContext
+ synapse.events.snapshot.EventContext:
"""
context = EventContext()
View
41 synapse/storage/_base.py
@@ -840,6 +840,47 @@ def _simple_delete_txn(txn, table, keyvalues):
return txn.execute(sql, keyvalues.values())
+ def _simple_delete_many(self, table, column, iterable, keyvalues, desc):
+ return self.runInteraction(
+ desc, self._simple_delete_many_txn, table, column, iterable, keyvalues
+ )
+
+ @staticmethod
+ def _simple_delete_many_txn(txn, table, column, iterable, keyvalues):
+ """Executes a DELETE query on the named table.
+
+ Filters rows by if value of `column` is in `iterable`.
+
+ Args:
+ txn : Transaction object
+ table : string giving the table name
+ column : column name to test for inclusion against `iterable`
+ iterable : list
+ keyvalues : dict of column names and values to select the rows with
+ """
+ if not iterable:
+ return
+
+ sql = "DELETE FROM %s" % table
+
+ clauses = []
+ values = []
+ clauses.append(
+ "%s IN (%s)" % (column, ",".join("?" for _ in iterable))
+ )
+ values.extend(iterable)
+
+ for key, value in keyvalues.items():
+ clauses.append("%s = ?" % (key,))
+ values.append(value)
+
+ if clauses:
+ sql = "%s WHERE %s" % (
+ sql,
+ " AND ".join(clauses),
+ )
+ return txn.execute(sql, values)
+
def _get_cache_dict(self, db_conn, table, entity_column, stream_column,
max_value, limit=100000):
# Fetch a mapping of room_id -> max stream position for "recent" rooms.
View
15 synapse/storage/background_updates.py
@@ -12,6 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+import synapse.util.async
from ._base import SQLBaseStore
from . import engines
@@ -84,24 +85,14 @@ def __init__(self, hs):
self._background_update_performance = {}
self._background_update_queue = []
self._background_update_handlers = {}
- self._background_update_timer = None
@defer.inlineCallbacks
def start_doing_background_updates(self):
- assert self._background_update_timer is None, \
- "background updates already running"
-
logger.info("Starting background schema updates")
while True:
- sleep = defer.Deferred()
- self._background_update_timer = self._clock.call_later(
- self.BACKGROUND_UPDATE_INTERVAL_MS / 1000., sleep.callback, None
- )
- try:
- yield sleep
- finally:
- self._background_update_timer = None
+ yield synapse.util.async.sleep(
+ self.BACKGROUND_UPDATE_INTERVAL_MS / 1000.)
try:
result = yield self.do_next_background_update(
View
6 synapse/storage/deviceinbox.py
@@ -357,12 +357,12 @@ def get_new_device_msgs_for_remote(
"""
Args:
destination(str): The name of the remote server.
- last_stream_id(int): The last position of the device message stream
+ last_stream_id(int|long): The last position of the device message stream
that the server sent up to.
- current_stream_id(int): The current position of the device
+ current_stream_id(int|long): The current position of the device
message stream.
Returns:
- Deferred ([dict], int): List of messages for the device and where
+ Deferred ([dict], int|long): List of messages for the device and where
in the stream the messages got to.
"""
View
19 synapse/storage/devices.py
@@ -108,6 +108,23 @@ def delete_device(self, user_id, device_id):
desc="delete_device",
)
+ def delete_devices(self, user_id, device_ids):
+ """Deletes several devices.
+
+ Args:
+ user_id (str): The ID of the user which owns the devices
+ device_ids (list): The IDs of the devices to delete
+ Returns:
+ defer.Deferred
+ """
+ return self._simple_delete_many(
+ table="devices",
+ column="device_id",
+ iterable=device_ids,
+ keyvalues={"user_id": user_id},
+ desc="delete_devices",
+ )
+
def update_device(self, user_id, device_id, new_display_name=None):
"""Update a device.
@@ -291,7 +308,7 @@ def get_devices_by_remote(self, destination, from_stream_id):
"""Get stream of updates to send to remote servers
Returns:
- (now_stream_id, [ { updates }, .. ])
+ (int, list[dict]): current stream id and list of updates
"""
now_stream_id = self._device_list_id_gen.get_current_token()
View
24 synapse/storage/event_federation.py
@@ -201,19 +201,19 @@ def _get_min_depth_interaction(self, txn, room_id):
def _update_min_depth_for_room_txn(self, txn, room_id, depth):
min_depth = self._get_min_depth_interaction(txn, room_id)
- do_insert = depth < min_depth if min_depth else True
+ if min_depth and depth >= min_depth:
+ return
- if do_insert:
- self._simple_upsert_txn(
- txn,
- table="room_depth",
- keyvalues={
- "room_id": room_id,
- },
- values={
- "min_depth": depth,
- },
- )
+ self._simple_upsert_txn(
+ txn,
+ table="room_depth",
+ keyvalues={
+ "room_id": room_id,
+ },
+ values={
+ "min_depth": depth,
+ },
+ )
def _handle_mult_prev_events(self, txn, events):
"""
View
361 synapse/storage/events.py

Large diffs are not rendered by default.

Oops, something went wrong.
View
13 synapse/storage/roommember.py
@@ -274,24 +274,27 @@ def _get_members_rows_txn(self, txn, room_id, membership=None, user_id=None):
return rows
- @cached(max_entries=500000, iterable=True)
+ @cachedInlineCallbacks(max_entries=500000, iterable=True)
def get_rooms_for_user(self, user_id):
- return self.get_rooms_for_user_where_membership_is(
+ """Returns a set of room_ids the user is currently joined to
+ """
+ rooms = yield self.get_rooms_for_user_where_membership_is(
user_id, membership_list=[Membership.JOIN],
)
+ defer.returnValue(frozenset(r.room_id for r in rooms))
@cachedInlineCallbacks(max_entries=500000, cache_context=True, iterable=True)
def get_users_who_share_room_with_user(self, user_id, cache_context):
"""Returns the set of users who share a room with `user_id`
"""
- rooms = yield self.get_rooms_for_user(
+ room_ids = yield self.get_rooms_for_user(
user_id, on_invalidate=cache_context.invalidate,
)
user_who_share_room = set()
- for room in rooms:
+ for room_id in room_ids:
user_ids = yield self.get_users_in_room(
- room.room_id, on_invalidate=cache_context.invalidate,
+ room_id, on_invalidate=cache_context.invalidate,
)
user_who_share_room.update(user_ids)
View
24 synapse/storage/state.py
@@ -14,7 +14,7 @@
# limitations under the License.
from ._base import SQLBaseStore
-from synapse.util.caches.descriptors import cached, cachedList
+from synapse.util.caches.descriptors import cached, cachedList, cachedInlineCallbacks
from synapse.util.caches import intern_string
from synapse.storage.engines import PostgresEngine
@@ -69,6 +69,18 @@ def __init__(self, hs):
where_clause="type='m.room.member'",
)
+ @cachedInlineCallbacks(max_entries=100000, iterable=True)
+ def get_current_state_ids(self, room_id):
+ rows = yield self._simple_select_list(
+ table="current_state_events",
+ keyvalues={"room_id": room_id},
+ retcols=["event_id", "type", "state_key"],
+ desc="_calculate_state_delta",
+ )
+ defer.returnValue({
+ (r["type"], r["state_key"]): r["event_id"] for r in rows
+ })
+
@defer.inlineCallbacks
def get_state_groups_ids(self, room_id, event_ids):
if not event_ids:
@@ -124,6 +136,16 @@ def _store_mult_state_groups_txn(self, txn, events_and_contexts):
continue
if context.current_state_ids is None:
+ # AFAIK, this can never happen
+ logger.error(
+ "Non-outlier event %s had current_state_ids==None",
+ event.event_id)
+ continue
+
+ # if the event was rejected, just give it the same state as its
+ # predecessor.
+ if context.rejected:
+ state_groups[event.event_id] = context.prev_group
continue
state_groups[event.event_id] = context.state_group
View
3 synapse/storage/stream.py
@@ -829,3 +829,6 @@ def update_federation_out_pos(self, typ, stream_id):
updatevalues={"stream_id": stream_id},
desc="update_federation_out_pos",
)
+
+ def has_room_changed_since(self, room_id, stream_id):
+ return self._events_stream_cache.has_entity_changed(room_id, stream_id)
View
14 synapse/storage/util/id_generators.py
@@ -30,6 +30,17 @@ def get_next(self):
def _load_current_id(db_conn, table, column, step=1):
+ """
+
+ Args:
+ db_conn (object):
+ table (str):
+ column (str):
+ step (int):
+
+ Returns:
+ int
+ """
cur = db_conn.cursor()
if step == 1:
cur.execute("SELECT MAX(%s) FROM %s" % (column, table,))
@@ -131,6 +142,9 @@ def manager():
def get_current_token(self):
"""Returns the maximum stream id such that all stream ids less than or
equal to it have been successfully persisted.
+
+ Returns:
+ int
"""
with self._lock:
if self._unfinished_ids:
View
2 synapse/util/caches/stream_change_cache.py
@@ -50,7 +50,7 @@ def __init__(self, name, current_stream_pos, max_size=10000, prefilled_cache={})
def has_entity_changed(self, entity, stream_pos):
"""Returns True if the entity may have been updated since stream_pos
"""
- assert type(stream_pos) is int
+ assert type(stream_pos) is int or type(stream_pos) is long
if stream_pos < self._earliest_known_stream_pos:
self.metrics.inc_misses()
View
54 synapse/util/logcontext.py
@@ -12,6 +12,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+""" Thread-local-alike tracking of log contexts within synapse
+
+This module provides objects and utilities for tracking contexts through
+synapse code, so that log lines can include a request identifier, and so that
+CPU and database activity can be accounted for against the request that caused
+them.
+
+See doc/log_contexts.rst for details on how this works.
+"""
+
from twisted.internet import defer
import threading
@@ -309,21 +319,43 @@ def preserve_context_over_deferred(deferred, context=None):
def preserve_fn(f):
- """Ensures that function is called with correct context and that context is
- restored after return. Useful for wrapping functions that return a deferred
- which you don't yield on.
+ """Wraps a function, to ensure that the current context is restored after
+ return from the function, and that the sentinel context is set once the
+ deferred returned by the funtion completes.
+
+ Useful for wrapping functions that return a deferred which you don't yield
+ on.
"""
+ def reset_context(result):
+ LoggingContext.set_current_context(LoggingContext.sentinel)
+ return result
+
+ # XXX: why is this here rather than inside g? surely we want to preserve
+ # the context from the time the function was called, not when it was
+ # wrapped?
current = LoggingContext.current_context()
def g(*args, **kwargs):
- with PreserveLoggingContext(current):
- res = f(*args, **kwargs)
- if isinstance(res, defer.Deferred):
- return preserve_context_over_deferred(
- res, context=LoggingContext.sentinel
- )
- else:
- return res
+ res = f(*args, **kwargs)
+ if isinstance(res, defer.Deferred) and not res.called:
+ # The function will have reset the context before returning, so
+ # we need to restore it now.
+ LoggingContext.set_current_context(current)
+
+ # The original context will be restored when the deferred
+ # completes, but there is nothing waiting for it, so it will
+ # get leaked into the reactor or some other function which
+ # wasn't expecting it. We therefore need to reset the context
+ # here.
+ #
+ # (If this feels asymmetric, consider it this way: we are
+ # effectively forking a new thread of execution. We are
+ # probably currently within a ``with LoggingContext()`` block,
+ # which is supposed to have a single entry and exit point. But
+ # by spawning off another deferred, we are effectively
+ # adding a new exit point.)
+ res.addBoth(reset_context)
+ return res
return g
View
40 synapse/util/msisdn.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Vector Creations Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import phonenumbers
+from synapse.api.errors import SynapseError
+
+
+def phone_number_to_msisdn(country, number):
+ """
+ Takes an ISO-3166-1 2 letter country code and phone number and
+ returns an msisdn representing the canonical version of that
+ phone number.
+ Args:
+ country (str): ISO-3166-1 2 letter country code
+ number (str): Phone number in a national or international format
+
+ Returns:
+ (str) The canonical form of the phone number, as an msisdn
+ Raises:
+ SynapseError if the number could not be parsed.
+ """
+ try:
+ phoneNumber = phonenumbers.parse(number, country)
+ except phonenumbers.NumberParseException:
+ raise SynapseError(400, "Unable to parse phone number")
+ return phonenumbers.format_number(
+ phoneNumber, phonenumbers.PhoneNumberFormat.E164
+ )[1:]
View
2 tests/handlers/test_presence.py
@@ -324,7 +324,7 @@ def test_sync_timeout(self):
state = UserPresenceState.default(user_id)
state = state.copy_and_replace(
state=PresenceState.ONLINE,
- last_active_ts=now,
+ last_active_ts=0,
last_user_sync_ts=now - SYNC_ONLINE_TIMEOUT - 1,
)
View
2 tests/replication/test_resource.py
@@ -68,7 +68,7 @@ def test_events(self):
code, body = yield get
self.assertEquals(code, 200)
self.assertEquals(body["events"]["field_names"], [
- "position", "internal", "json", "state_group"
+ "position", "event_id", "room_id", "type", "state_key",
])
@defer.inlineCallbacks
View
61 tests/util/test_log_context.py
@@ -1,8 +1,10 @@
+import twisted.python.failure
from twisted.internet import defer
from twisted.internet import reactor
from .. import unittest
from synapse.util.async import sleep
+from synapse.util import logcontext
from synapse.util.logcontext import LoggingContext
@@ -33,3 +35,62 @@ def competing_callback():
context_one.test_key = "one"
yield sleep(0)
self._check_test_key("one")
+
+ def _test_preserve_fn(self, function):
+ sentinel_context = LoggingContext.current_context()
+
+ callback_completed = [False]
+
+ @defer.inlineCallbacks
+ def cb():
+ context_one.test_key = "one"
+ yield function()
+ self._check_test_key("one")
+
+ callback_completed[0] = True
+
+ with LoggingContext() as context_one:
+ context_one.test_key = "one"
+
+ # fire off function, but don't wait on it.
+ logcontext.preserve_fn(cb)()
+
+ self._check_test_key("one")
+
+ # now wait for the function under test to have run, and check that
+ # the logcontext is left in a sane state.
+ d2 = defer.Deferred()
+
+ def check_logcontext():
+ if not callback_completed[0]:
+ reactor.callLater(0.01, check_logcontext)
+ return
+
+ # make sure that the context was reset before it got thrown back
+ # into the reactor
+ try:
+ self.assertIs(LoggingContext.current_context(),
+ sentinel_context)
+ d2.callback(None)
+ except BaseException:
+ d2.errback(twisted.python.failure.Failure())
+
+ reactor.callLater(0.01, check_logcontext)
+
+ # test is done once d2 finishes
+ return d2
+
+ def test_preserve_fn_with_blocking_fn(self):
+ @defer.inlineCallbacks
+ def blocking_function():
+ yield sleep(0)
+
+ return self._test_preserve_fn(blocking_function)
+
+ def test_preserve_fn_with_non_blocking_fn(self):
+ @defer.inlineCallbacks
+ def nonblocking_function():
+ with logcontext.PreserveLoggingContext():
+ yield defer.succeed(None)
+
+ return self._test_preserve_fn(nonblocking_function)