From 6baeda9ab7df685ebd2d6f4bb40a8a0af4dfea34 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Wed, 17 Aug 2022 23:29:36 +0200 Subject: [PATCH 01/13] Move the older dehydrated devices behind a config flag --- synapse/config/experimental.py | 4 ++++ synapse/rest/client/devices.py | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index c1ff41753994..fc5d959432e8 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -35,6 +35,10 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: # MSC2285 (unstable private read receipts) self.msc2285_enabled: bool = experimental.get("msc2285_enabled", False) + # MSC2697 (device dehydration) + # Enabled by default since this option was added after adding the feature. + self.msc2697_enabled: bool = experimental.get("msc2697_enabled", True) + # MSC3244 (room version capabilities) self.msc3244_enabled: bool = experimental.get("msc3244_enabled", True) diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index ed6ce78d4771..ab6149f0c6c4 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -331,5 +331,6 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: DeleteDevicesRestServlet(hs).register(http_server) DevicesRestServlet(hs).register(http_server) DeviceRestServlet(hs).register(http_server) - DehydratedDeviceServlet(hs).register(http_server) - ClaimDehydratedDeviceServlet(hs).register(http_server) + if hs.config.experimental.msc2697_enabled: + DehydratedDeviceServlet(hs).register(http_server) + ClaimDehydratedDeviceServlet(hs).register(http_server) From ace4f49aeb9ec8344059b378bc15c957215eff76 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Thu, 18 Aug 2022 19:23:53 +0200 Subject: [PATCH 02/13] Implement MSC3814 Some differences: - We use GET instead of POST for the events endpoint. - We don't delete the device implicitly when fetching events. - We allow the fetch device messages endpoint for both dehydrated and the current requesters device. Signed-off-by: Nicolas Werner --- synapse/config/experimental.py | 4 ++ synapse/handlers/devicemessage.py | 69 +++++++++++++++++++- synapse/rest/client/devices.py | 53 +++++++++++++-- tests/handlers/test_device.py | 104 ++++++++++++++++++++++++++++++ 4 files changed, 223 insertions(+), 7 deletions(-) diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index fc5d959432e8..e1ea65b69592 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -39,6 +39,10 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: # Enabled by default since this option was added after adding the feature. self.msc2697_enabled: bool = experimental.get("msc2697_enabled", True) + # MSC3814 (dehydrated devices with SSSS) + # This is an alternative method to achieve the same goals as MSC2697. + self.msc3814_enabled: bool = experimental.get("msc3814_enabled", False) + # MSC3244 (room version capabilities) self.msc3244_enabled: bool = experimental.get("msc3244_enabled", True) diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index 444c08bc2eef..39cd0084b630 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -13,10 +13,10 @@ # limitations under the License. import logging -from typing import TYPE_CHECKING, Any, Dict +from typing import TYPE_CHECKING, Any, Dict, Optional from synapse.api.constants import EduTypes, ToDeviceEventTypes -from synapse.api.errors import SynapseError +from synapse.api.errors import Codes, SynapseError from synapse.api.ratelimiting import Ratelimiter from synapse.logging.context import run_in_background from synapse.logging.opentracing import ( @@ -46,6 +46,9 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.notifier = hs.get_notifier() self.is_mine = hs.is_mine + if hs.config.experimental.msc3814_enabled: + self.event_sources = hs.get_event_sources() + self.device_handler = hs.get_device_handler() # We only need to poke the federation sender explicitly if its on the # same instance. Other federation sender instances will get notified by @@ -293,3 +296,65 @@ async def send_device_message( # Enqueue a new federation transaction to send the new # device messages to each remote destination. self.federation_sender.send_device_messages(destination) + + async def get_events_for_dehydrated_device( + self, + requester: Requester, + device_id: str, + since_token: Optional[str], + limit: int, + ) -> JsonDict: + """Fetches up to `limit` events sent to `device_id` starting from `since_token` and returns the new since token.""" + + user_id = requester.user.to_string() + + # TODO(Nico): Figure out who should be allowed to use that endpoint. + # For now we just allow it for yourself and for the dehydrated device. + if device_id != requester.device_id: + dehydrated_device = await self.device_handler.get_dehydrated_device(user_id) + if dehydrated_device is not None and device_id != dehydrated_device[0]: + raise SynapseError( + 403, + "Can only fetch messages for own device or dehydrated devices", + Codes.UNAUTHORIZED, + ) + + since_stream_id = 0 + if since_token and len(since_token) > 1 and since_token[0] == "d": + since_stream_id = int(since_token[1:]) + + # if we have a since token, delete any to-device messages before that token + # (since we now know that the device has received them) + deleted = await self.store.delete_messages_for_device( + user_id, device_id, since_stream_id + ) + logger.debug( + "Deleted %d to-device messages up to %d", deleted, since_stream_id + ) + + to_token = self.event_sources.get_current_token().to_device_key + + messages, stream_id = await self.store.get_messages_for_device( + user_id, device_id, since_stream_id, to_token, limit + ) + + for message in messages: + # We pop here as we shouldn't be sending the message ID down + # `/sync` + message_id = message.pop("message_id", None) + if message_id: + set_tag(SynapseTags.TO_DEVICE_MESSAGE_ID, message_id) + + logger.debug( + "Returning %d to-device messages between %d and %d (current token: %d) for dehydrated device %s", + len(messages), + since_stream_id, + stream_id, + to_token, + device_id, + ) + + return { + "events": messages, + "next_batch": f"d{stream_id}", + } diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index ab6149f0c6c4..8600bf5718d1 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -18,11 +18,13 @@ from synapse.api import errors from synapse.api.errors import NotFoundError -from synapse.http.server import HttpServer +from synapse.http.server import HttpServer, cancellable from synapse.http.servlet import ( RestServlet, assert_params_in_dict, + parse_integer, parse_json_object_from_request, + parse_string, ) from synapse.http.site import SynapseRequest from synapse.rest.client._base import client_patterns, interactive_auth_handler @@ -194,6 +196,8 @@ async def on_PUT( class DehydratedDeviceServlet(RestServlet): """Retrieve or store a dehydrated device. + Implements both MSC2697 and MSC3814. + GET /org.matrix.msc2697.v2/dehydrated_device HTTP/1.1 200 OK @@ -226,14 +230,19 @@ class DehydratedDeviceServlet(RestServlet): """ - PATTERNS = client_patterns("/org.matrix.msc2697.v2/dehydrated_device", releases=()) - - def __init__(self, hs: "HomeServer"): + def __init__(self, hs: "HomeServer", msc2697: bool = True): super().__init__() self.hs = hs self.auth = hs.get_auth() self.device_handler = hs.get_device_handler() + self.PATTERNS = client_patterns( + "/org.matrix.msc2697.v2/dehydrated_device" + if msc2697 + else "/org.matrix.msc3814.v1/dehydrated_device$", + releases=(), + ) + async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request) dehydrated_device = await self.device_handler.get_dehydrated_device( @@ -327,10 +336,44 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: return 200, result +class DehydratedDeviceEventsServlet(RestServlet): + PATTERNS = client_patterns( + "/org.matrix.msc3814.v1/dehydrated_device/(?P[^/]*)/events$", + releases=(), + ) + + def __init__(self, hs: "HomeServer"): + super().__init__() + self.message_handler = hs.get_device_message_handler() + self.auth = hs.get_auth() + self.store = hs.get_datastores().main + + @cancellable + async def on_GET( + self, request: SynapseRequest, device_id: str + ) -> Tuple[int, JsonDict]: + requester = await self.auth.get_user_by_req(request) + + from_tok = parse_string(request, "from") + limit = parse_integer(request, "limit", 100) + + msgs = await self.message_handler.get_events_for_dehydrated_device( + requester=requester, + device_id=device_id, + since_token=from_tok, + limit=limit, + ) + + return 200, msgs + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: DeleteDevicesRestServlet(hs).register(http_server) DevicesRestServlet(hs).register(http_server) DeviceRestServlet(hs).register(http_server) if hs.config.experimental.msc2697_enabled: - DehydratedDeviceServlet(hs).register(http_server) + DehydratedDeviceServlet(hs, msc2697=True).register(http_server) ClaimDehydratedDeviceServlet(hs).register(http_server) + if hs.config.experimental.msc3814_enabled: + DehydratedDeviceServlet(hs, msc2697=False).register(http_server) + DehydratedDeviceEventsServlet(hs).register(http_server) diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index b8b465d35b8f..de3cccb5d6d5 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -16,11 +16,13 @@ from typing import Optional +from twisted.internet.defer import ensureDeferred from twisted.test.proto_helpers import MemoryReactor from synapse.api.errors import NotFoundError, SynapseError from synapse.handlers.device import MAX_DEVICE_DISPLAY_NAME_LEN from synapse.server import HomeServer +from synapse.types import create_requester from synapse.util import Clock from tests import unittest @@ -265,6 +267,7 @@ class DehydrationTestCase(unittest.HomeserverTestCase): def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: hs = self.setup_test_homeserver("server", federation_http_client=None) self.handler = hs.get_device_handler() + self.message_handler = hs.get_device_message_handler() self.registration = hs.get_registration_handler() self.auth = hs.get_auth() self.store = hs.get_datastores().main @@ -342,3 +345,104 @@ def test_dehydrate_and_rehydrate_device(self) -> None: ret = self.get_success(self.handler.get_dehydrated_device(user_id=user_id)) self.assertIsNone(ret) + + @unittest.override_config( + {"experimental_features": {"msc2697_enabled": False, "msc3814_enabled": True}} + ) + def test_dehydrate_v2_and_fetch_events(self) -> None: + user_id = "@boris:server" + + self.get_success(self.store.register_user(user_id, "foobar")) + + # First check if we can store and fetch a dehydrated device + stored_dehydrated_device_id = self.get_success( + self.handler.store_dehydrated_device( + user_id=user_id, + device_data={"device_data": {"foo": "bar"}}, + initial_device_display_name="dehydrated device", + ) + ) + + retrieved_device_id, device_data = self.get_success( + self.handler.get_dehydrated_device(user_id=user_id) + ) + + self.assertEqual(retrieved_device_id, stored_dehydrated_device_id) + self.assertEqual(device_data, {"device_data": {"foo": "bar"}}) + + # Create a new login for the user + device_id, access_token, _expiration_time, _refresh_token = self.get_success( + self.registration.register_device( + user_id=user_id, + device_id=None, + initial_display_name="new device", + ) + ) + + requester = create_requester(user_id, device_id=device_id) + + # Fetching messages for a non existing device should return an error + self.get_failure( + self.message_handler.get_events_for_dehydrated_device( + requester=requester, + device_id="not the right device ID", + since_token=None, + limit=10, + ), + SynapseError, + ) + + # Send a message to the dehydrated device + ensureDeferred( + self.message_handler.send_device_message( + requester=requester, + message_type="test.message", + messages={user_id: {stored_dehydrated_device_id: {"body": "foo"}}}, + ) + ) + self.pump() + + # Fetch the message of the dehydrated device + res = self.get_success( + self.message_handler.get_events_for_dehydrated_device( + requester=requester, + device_id=stored_dehydrated_device_id, + since_token=None, + limit=10, + ) + ) + + self.assertTrue(len(res["next_batch"]) > 1) + self.assertEqual(len(res["events"]), 1) + self.assertEqual(res["events"][0]["content"]["body"], "foo") + + # Fetch the message of the dehydrated device again, which should return nothing and delete the old messages + res = self.get_success( + self.message_handler.get_events_for_dehydrated_device( + requester=requester, + device_id=stored_dehydrated_device_id, + since_token=res["next_batch"], + limit=10, + ) + ) + self.assertTrue(len(res["next_batch"]) > 1) + self.assertEqual(len(res["events"]), 0) + + # Fetching messages without since should return nothing, since the messages got deleted + res = self.get_success( + self.message_handler.get_events_for_dehydrated_device( + requester=requester, + device_id=stored_dehydrated_device_id, + since_token=None, + limit=10, + ) + ) + self.assertTrue(len(res["next_batch"]) > 1) + self.assertEqual(len(res["events"]), 0) + + # We don't delete the device when fetch messages for now. + # # make sure that the device ID that we were initially assigned no longer exists + # self.get_failure( + # self.handler.get_device(user_id, device_id), + # NotFoundError, + # ) From 49f892db8d10b9bee5a94d257b430eed4ebbce53 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Mon, 22 Aug 2022 18:04:22 +0200 Subject: [PATCH 03/13] Add changelog --- changelog.d/13581.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/13581.feature diff --git a/changelog.d/13581.feature b/changelog.d/13581.feature new file mode 100644 index 000000000000..1db4ab71d361 --- /dev/null +++ b/changelog.d/13581.feature @@ -0,0 +1 @@ +Implement [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814), dehydrated devices v2/shrivelled sessions, with a few changes (as proposed on the MSC) and move [MSC2697](https://github.com/matrix-org/matrix-spec-proposals/pull/2697) behind a config flag. Contributed by Nico from Famedly. From 6c183c53f27f2780f4e63a7f4c9729aaa313db29 Mon Sep 17 00:00:00 2001 From: Nicolas Werner Date: Thu, 25 Aug 2022 15:10:08 +0200 Subject: [PATCH 04/13] Address review feedback Return errors on invalid token formats and terminate regex in both cases --- synapse/handlers/devicemessage.py | 21 ++++++++++++++++++--- synapse/rest/client/devices.py | 2 +- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index 39cd0084b630..4eecaba926f7 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -13,6 +13,7 @@ # limitations under the License. import logging +from http import HTTPStatus from typing import TYPE_CHECKING, Any, Dict, Optional from synapse.api.constants import EduTypes, ToDeviceEventTypes @@ -314,14 +315,28 @@ async def get_events_for_dehydrated_device( dehydrated_device = await self.device_handler.get_dehydrated_device(user_id) if dehydrated_device is not None and device_id != dehydrated_device[0]: raise SynapseError( - 403, + HTTPStatus.FORBIDDEN, "Can only fetch messages for own device or dehydrated devices", Codes.UNAUTHORIZED, ) since_stream_id = 0 - if since_token and len(since_token) > 1 and since_token[0] == "d": - since_stream_id = int(since_token[1:]) + if since_token: + if not since_token.startswith("d"): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "from parameter %r has an invalid format" % (since_token,), + errcode=Codes.INVALID_PARAM, + ) + + try: + since_stream_id = int(since_token[1:]) + except Exception: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "from parameter %r has an invalid format" % (since_token,), + errcode=Codes.INVALID_PARAM, + ) # if we have a since token, delete any to-device messages before that token # (since we now know that the device has received them) diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index 8600bf5718d1..c5260aff28a8 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -237,7 +237,7 @@ def __init__(self, hs: "HomeServer", msc2697: bool = True): self.device_handler = hs.get_device_handler() self.PATTERNS = client_patterns( - "/org.matrix.msc2697.v2/dehydrated_device" + "/org.matrix.msc2697.v2/dehydrated_device$" if msc2697 else "/org.matrix.msc3814.v1/dehydrated_device$", releases=(), From 6d0ce6f9fd9e6129a74f99f3781ae45fb4c46540 Mon Sep 17 00:00:00 2001 From: Nicolas Werner <89468146+nico-famedly@users.noreply.github.com> Date: Tue, 6 Sep 2022 10:59:07 +0200 Subject: [PATCH 05/13] Update synapse/handlers/devicemessage.py Co-authored-by: Hubert Chathi --- synapse/handlers/devicemessage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index 4eecaba926f7..7419e7e62a07 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -317,7 +317,7 @@ async def get_events_for_dehydrated_device( raise SynapseError( HTTPStatus.FORBIDDEN, "Can only fetch messages for own device or dehydrated devices", - Codes.UNAUTHORIZED, + Codes.FORBIDDEN, ) since_stream_id = 0 From b20c4c7691a21e34655345c9b48cf5a16d8d0702 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 12 Jul 2023 11:44:44 -0700 Subject: [PATCH 06/13] bring into alignment with msc3814 --- synapse/handlers/devicemessage.py | 39 ++++++++++++++++++++----------- synapse/rest/client/devices.py | 13 +++++++---- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index 734a71f5bcb8..621f4c4449c1 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -16,7 +16,6 @@ from http import HTTPStatus from typing import TYPE_CHECKING, Any, Dict, Optional - from synapse.api.constants import EduTypes, EventContentFields, ToDeviceEventTypes from synapse.api.errors import Codes, SynapseError from synapse.api.ratelimiting import Ratelimiter @@ -316,20 +315,28 @@ async def get_events_for_dehydrated_device( since_token: Optional[str], limit: int, ) -> JsonDict: - """Fetches up to `limit` events sent to `device_id` starting from `since_token` and returns the new since token.""" + """Fetches up to `limit` events sent to `device_id` starting from `since_token` + and returns the new since token. If there are no more messages, returns an empty + array and deletes the dehydrated device associated with the user/device_id. + + Args: + requester: the user requesting the messages + device_id: ID of the dehydrated device + since_token: stream id to start from when fetching messages + limit: the number of messages to fetch + """ user_id = requester.user.to_string() - # TODO(Nico): Figure out who should be allowed to use that endpoint. - # For now we just allow it for yourself and for the dehydrated device. - if device_id != requester.device_id: - dehydrated_device = await self.device_handler.get_dehydrated_device(user_id) - if dehydrated_device is not None and device_id != dehydrated_device[0]: - raise SynapseError( - HTTPStatus.FORBIDDEN, - "Can only fetch messages for own device or dehydrated devices", - Codes.FORBIDDEN, - ) + # only allow fetching messages for the dehydrated device id currently associated + # with the user + dehydrated_device = await self.device_handler.get_dehydrated_device(user_id) + if dehydrated_device is None or device_id != dehydrated_device[0]: + raise SynapseError( + HTTPStatus.FORBIDDEN, + "You may only fetch messages for your dehydrated device", + Codes.FORBIDDEN, + ) since_stream_id = 0 if since_token: @@ -369,7 +376,7 @@ async def get_events_for_dehydrated_device( # `/sync` message_id = message.pop("message_id", None) if message_id: - set_tag(SynapseTags.TO_DEVICE_MESSAGE_ID, message_id) + set_tag(SynapseTags.TO_DEVICE_EDU_ID, message_id) logger.debug( "Returning %d to-device messages between %d and %d (current token: %d) for dehydrated device %s", @@ -380,6 +387,12 @@ async def get_events_for_dehydrated_device( device_id, ) + if messages == []: + # we've fetched all the messages, delete the dehydrated device + await self.store.remove_dehydrated_device( + requester.user.to_string(), device_id + ) + return { "events": messages, "next_batch": f"d{stream_id}", diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index e3434b68d561..b33c731fd7e0 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -19,7 +19,6 @@ from pydantic import Extra, StrictStr from synapse.api import errors - from synapse.api.errors import NotFoundError, UnrecognizedRequestError from synapse.handlers.device import DeviceHandler from synapse.http.server import HttpServer @@ -27,7 +26,6 @@ RestServlet, parse_and_validate_json_object_from_request, parse_integer, - parse_string, ) from synapse.http.site import SynapseRequest from synapse.rest.client._base import client_patterns, interactive_auth_handler @@ -370,19 +368,24 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.store = hs.get_datastores().main + class PostBody(RequestBodyModel): + next_batch: Optional[StrictStr] + @cancellable - async def on_GET( + async def on_POST( self, request: SynapseRequest, device_id: str ) -> Tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request) - from_tok = parse_string(request, "from") + next_batch = parse_and_validate_json_object_from_request( + request, self.PostBody + ).next_batch limit = parse_integer(request, "limit", 100) msgs = await self.message_handler.get_events_for_dehydrated_device( requester=requester, device_id=device_id, - since_token=from_tok, + since_token=next_batch, limit=limit, ) From a950d5ecad285c555f24f4fe65258af4e0a5a06b Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 12 Jul 2023 12:21:11 -0700 Subject: [PATCH 07/13] fix merge issues --- synapse/config/experimental.py | 10 ++++------ synapse/rest/client/devices.py | 1 - 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 67ea1b9ed52b..e8cd1e68c8c1 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -247,18 +247,16 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: # MSC3026 (busy presence state) self.msc3026_enabled: bool = experimental.get("msc3026_enabled", False) - # MSC2716 (importing historical messages) - self.msc2716_enabled: bool = experimental.get("msc2716_enabled", False) - - # MSC2285 (unstable private read receipts) - self.msc2285_enabled: bool = experimental.get("msc2285_enabled", False) - # MSC2697 (device dehydration) # Enabled by default since this option was added after adding the feature. + # It is not recommended that both MSC2697 and MSC3814 both be enabled at + # once. self.msc2697_enabled: bool = experimental.get("msc2697_enabled", True) # MSC3814 (dehydrated devices with SSSS) # This is an alternative method to achieve the same goals as MSC2697. + # It is not recommended that both MSC2697 and MSC3814 both be enabled at + # once. self.msc3814_enabled: bool = experimental.get("msc3814_enabled", False) # MSC3244 (room version capabilities) diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index b33c731fd7e0..ace97b519fc0 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -399,7 +399,6 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: ): DeleteDevicesRestServlet(hs).register(http_server) DevicesRestServlet(hs).register(http_server) - DeviceRestServlet(hs).register(http_server) if hs.config.experimental.msc2697_enabled: DehydratedDeviceServlet(hs, msc2697=True).register(http_server) ClaimDehydratedDeviceServlet(hs).register(http_server) From fe9be3c67a7dfd206772c2279878fe330360be91 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 12 Jul 2023 12:21:17 -0700 Subject: [PATCH 08/13] add some tests --- tests/handlers/test_device.py | 38 +++++---- tests/rest/client/test_devices.py | 134 +++++++++++++++++++++++++++++- 2 files changed, 157 insertions(+), 15 deletions(-) diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index 2b0cac77ac50..1be366702b51 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -24,9 +24,11 @@ from synapse.api.errors import NotFoundError, SynapseError from synapse.appservice import ApplicationService from synapse.handlers.device import MAX_DEVICE_DISPLAY_NAME_LEN, DeviceHandler +from synapse.rest import admin +from synapse.rest.client import devices, login, register from synapse.server import HomeServer from synapse.storage.databases.main.appservice import _make_exclusive_regex -from synapse.types import create_requester, JsonDict +from synapse.types import JsonDict, create_requester from synapse.util import Clock from tests import unittest @@ -401,11 +403,19 @@ def test_on_federation_query_user_devices_appservice(self) -> None: class DehydrationTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets_for_client_rest_resource, + login.register_servlets, + register.register_servlets, + devices.register_servlets, + ] + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: hs = self.setup_test_homeserver("server", federation_http_client=None) handler = hs.get_device_handler() assert isinstance(handler, DeviceHandler) self.handler = handler + self.message_handler = hs.get_device_message_handler() self.registration = hs.get_registration_handler() self.auth = hs.get_auth() self.store = hs.get_datastores().main @@ -501,10 +511,11 @@ def test_dehydrate_v2_and_fetch_events(self) -> None: ) ) - retrieved_device_id, device_data = self.get_success( + device_info = self.get_success( self.handler.get_dehydrated_device(user_id=user_id) ) - + assert device_info is not None + retrieved_device_id, device_data = device_info self.assertEqual(retrieved_device_id, stored_dehydrated_device_id) self.assertEqual(device_data, {"device_data": {"foo": "bar"}}) @@ -566,21 +577,20 @@ def test_dehydrate_v2_and_fetch_events(self) -> None: self.assertTrue(len(res["next_batch"]) > 1) self.assertEqual(len(res["events"]), 0) - # Fetching messages without since should return nothing, since the messages got deleted - res = self.get_success( + # Fetching messages again should fail, since the messages and dehydrated device + # were deleted + self.get_failure( self.message_handler.get_events_for_dehydrated_device( requester=requester, device_id=stored_dehydrated_device_id, since_token=None, limit=10, - ) + ), + SynapseError, ) - self.assertTrue(len(res["next_batch"]) > 1) - self.assertEqual(len(res["events"]), 0) - # We don't delete the device when fetch messages for now. - # # make sure that the device ID that we were initially assigned no longer exists - # self.get_failure( - # self.handler.get_device(user_id, device_id), - # NotFoundError, - # ) + # make sure that the dehydrated device ID is deleted after fetching messages + res2 = self.get_success( + self.handler.get_dehydrated_device(requester.user.to_string()), + ) + self.assertEqual(res2, None) diff --git a/tests/rest/client/test_devices.py b/tests/rest/client/test_devices.py index d80eea17d3af..25e425b0fe6e 100644 --- a/tests/rest/client/test_devices.py +++ b/tests/rest/client/test_devices.py @@ -13,12 +13,14 @@ # limitations under the License. from http import HTTPStatus +from twisted.internet.defer import ensureDeferred from twisted.test.proto_helpers import MemoryReactor from synapse.api.errors import NotFoundError from synapse.rest import admin, devices, room, sync -from synapse.rest.client import account, login, register +from synapse.rest.client import account, keys, login, register from synapse.server import HomeServer +from synapse.types import JsonDict, create_requester from synapse.util import Clock from tests import unittest @@ -208,8 +210,13 @@ class DehydratedDeviceTestCase(unittest.HomeserverTestCase): login.register_servlets, register.register_servlets, devices.register_servlets, + keys.register_servlets, ] + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.registration = hs.get_registration_handler() + self.message_handler = hs.get_device_message_handler() + def test_PUT(self) -> None: """Sanity-check that we can PUT a dehydrated device. @@ -234,3 +241,128 @@ def test_PUT(self) -> None: self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) device_id = channel.json_body.get("device_id") self.assertIsInstance(device_id, str) + + @unittest.override_config( + {"experimental_features": {"msc2697_enabled": False, "msc3814_enabled": True}} + ) + def test_dehydrate_msc3814(self) -> None: + user = self.register_user("mikey", "pass") + token = self.login(user, "pass", device_id="device1") + content: JsonDict = { + "device_data": { + "algorithm": "m.dehydration.v1.olm", + }, + "initial_device_display_name": "foo bar", + } + channel = self.make_request( + "PUT", + "_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device", + content=content, + access_token=token, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + device_id = channel.json_body.get("device_id") + assert device_id is not None + self.assertIsInstance(device_id, str) + + # test that you can upload keys for this device + content = { + "device_keys": { + "algorithms": ["m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"], + "device_id": f"{device_id}", + "keys": { + "curve25519:JLAFKJWSCS": "3C5BFWi2Y8MaVvjM8M22DBmh24PmgR0nPvJOIArzgyI", + "ed25519:JLAFKJWSCS": "lEuiRJBit0IG6nUf5pUzWTUEsRVVe/HJkoKuEww9ULI", + }, + "signatures": { + "@alice:example.com": { + "ed25519:JLAFKJWSCS": "dSO80A01XiigH3uBiDVx/EjzaoycHcjq9lfQX0uWsqxl2giMIiSPR8a4d291W1ihKJL/a+myXS367WT6NAIcBA" + } + }, + "user_id": f"{user}", + }, + } + channel = self.make_request( + "POST", + f"/_matrix/client/r0/keys/upload/{device_id}", + content=content, + access_token=token, + ) + self.assertEqual(channel.code, 200) + + # test that we can now GET the dehydrated device info + channel = self.make_request( + "GET", + "_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device", + access_token=token, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + returned_device_id = channel.json_body.get("device_id") + self.assertEqual(returned_device_id, device_id) + device_data = channel.json_body.get("device_data") + expected_device_data = { + "algorithm": "m.dehydration.v1.olm", + } + self.assertEqual(device_data, expected_device_data) + + # create another device for the user + ( + new_device_id, + _, + _, + _, + ) = self.get_success( + self.registration.register_device( + user_id=user, + device_id=None, + initial_display_name="new device", + ) + ) + requester = create_requester(user, device_id=new_device_id) + + # Send a message to the dehydrated device + ensureDeferred( + self.message_handler.send_device_message( + requester=requester, + message_type="test.message", + messages={user: {device_id: {"body": "test_message"}}}, + ) + ) + self.pump() + + # make sure we can fetch the message with our dehydrated device id + channel = self.make_request( + "POST", + f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events", + content={}, + access_token=token, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + expected_content = {"body": "test_message"} + self.assertEqual(channel.json_body["events"][0]["content"], expected_content) + next_batch_token = channel.json_body.get("next_batch") + + # fetch messages again and make sure that the message was deleted and we are returned an + # empty array + content = {"next_batch": next_batch_token} + channel = self.make_request( + "POST", + f"_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device/{device_id}/events", + content=content, + access_token=token, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body["events"], []) + + # make sure that the dehydrated device id is deleted after we received the messages + channel = self.make_request( + "GET", + "_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device", + access_token=token, + shorthand=False, + ) + self.assertEqual(channel.code, 404) From ccd6c12b3873420d085fa2b22989b5133b683471 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 12 Jul 2023 12:21:25 -0700 Subject: [PATCH 09/13] newsfragment --- changelog.d/13581.feature | 1 - changelog.d/15929.feature | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/13581.feature create mode 100644 changelog.d/15929.feature diff --git a/changelog.d/13581.feature b/changelog.d/13581.feature deleted file mode 100644 index 1db4ab71d361..000000000000 --- a/changelog.d/13581.feature +++ /dev/null @@ -1 +0,0 @@ -Implement [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814), dehydrated devices v2/shrivelled sessions, with a few changes (as proposed on the MSC) and move [MSC2697](https://github.com/matrix-org/matrix-spec-proposals/pull/2697) behind a config flag. Contributed by Nico from Famedly. diff --git a/changelog.d/15929.feature b/changelog.d/15929.feature new file mode 100644 index 000000000000..f0814ec15dd9 --- /dev/null +++ b/changelog.d/15929.feature @@ -0,0 +1,3 @@ +Implement [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814), +dehydrated devices v2/shrivelled sessions and move [MSC2697](https://github.com/matrix-org/matrix-spec-proposals/pull/2697) +behind a config flag. Contributed by Nico from Famedly and H-Shay. From 664ad97bff2a1edce85583e2741ec8870851af5a Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Wed, 12 Jul 2023 14:22:44 -0700 Subject: [PATCH 10/13] fix running on workers --- synapse/rest/client/devices.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index ace97b519fc0..3223764464a4 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -399,12 +399,12 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: ): DeleteDevicesRestServlet(hs).register(http_server) DevicesRestServlet(hs).register(http_server) - if hs.config.experimental.msc2697_enabled: - DehydratedDeviceServlet(hs, msc2697=True).register(http_server) - ClaimDehydratedDeviceServlet(hs).register(http_server) - if hs.config.experimental.msc3814_enabled: - DehydratedDeviceServlet(hs, msc2697=False).register(http_server) - DehydratedDeviceEventsServlet(hs).register(http_server) if hs.config.worker.worker_app is None: DeviceRestServlet(hs).register(http_server) + if hs.config.experimental.msc2697_enabled: + DehydratedDeviceServlet(hs, msc2697=True).register(http_server) + ClaimDehydratedDeviceServlet(hs).register(http_server) + if hs.config.experimental.msc3814_enabled: + DehydratedDeviceServlet(hs, msc2697=False).register(http_server) + DehydratedDeviceEventsServlet(hs).register(http_server) From a52a25a4913f1c8e0b820028e2aaa516e6e6b118 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Fri, 14 Jul 2023 14:28:59 -0700 Subject: [PATCH 11/13] suggestions from code review --- changelog.d/15929.feature | 4 +--- synapse/config/experimental.py | 9 +++++++++ synapse/handlers/devicemessage.py | 10 +++++++++- synapse/rest/client/devices.py | 2 +- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/changelog.d/15929.feature b/changelog.d/15929.feature index f0814ec15dd9..c3aaeae66ee7 100644 --- a/changelog.d/15929.feature +++ b/changelog.d/15929.feature @@ -1,3 +1 @@ -Implement [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814), -dehydrated devices v2/shrivelled sessions and move [MSC2697](https://github.com/matrix-org/matrix-spec-proposals/pull/2697) -behind a config flag. Contributed by Nico from Famedly and H-Shay. +Implement [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814), dehydrated devices v2/shrivelled sessions and move [MSC2697](https://github.com/matrix-org/matrix-spec-proposals/pull/2697) behind a config flag. Contributed by Nico from Famedly and H-Shay. diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index e8cd1e68c8c1..1695ed8ca336 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -259,6 +259,15 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: # once. self.msc3814_enabled: bool = experimental.get("msc3814_enabled", False) + if self.msc2697_enabled and self.msc3814_enabled: + raise ConfigError( + "MSC2697 and MSC3814 should not both be enabled.", + ( + "experimental_features", + "msc3814_enabled", + ), + ) + # MSC3244 (room version capabilities) self.msc3244_enabled: bool = experimental.get("msc3244_enabled", True) diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index 621f4c4449c1..a541d9703cc2 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -331,7 +331,15 @@ async def get_events_for_dehydrated_device( # only allow fetching messages for the dehydrated device id currently associated # with the user dehydrated_device = await self.device_handler.get_dehydrated_device(user_id) - if dehydrated_device is None or device_id != dehydrated_device[0]: + if dehydrated_device is None: + raise SynapseError( + HTTPStatus.FORBIDDEN, + "You may only fetch messages for your dehydrated device", + Codes.FORBIDDEN, + ) + + dehydrated_device_id = dehydrated_device[0] + if device_id != dehydrated_device_id: raise SynapseError( HTTPStatus.FORBIDDEN, "You may only fetch messages for your dehydrated device", diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index 3223764464a4..89776f639263 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -231,7 +231,7 @@ class Config: class DehydratedDeviceServlet(RestServlet): """Retrieve or store a dehydrated device. - Implements both MSC2697 and MSC3814. + Implements either MSC2697 and MSC3814. GET /org.matrix.msc2697.v2/dehydrated_device From 0f49f816aafa1d169375e4d9048e0ea98e990e6b Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Mon, 17 Jul 2023 13:24:16 -0700 Subject: [PATCH 12/13] requested changes --- synapse/handlers/devicemessage.py | 10 ++++++---- synapse/rest/client/devices.py | 2 -- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index a541d9703cc2..92d4b7b206c4 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -324,6 +324,9 @@ async def get_events_for_dehydrated_device( device_id: ID of the dehydrated device since_token: stream id to start from when fetching messages limit: the number of messages to fetch + Returns: + A dict containing the to-device messages, as well as a token that the client + can provide in the next call to fetch the next batch of messages """ user_id = requester.user.to_string() @@ -334,11 +337,11 @@ async def get_events_for_dehydrated_device( if dehydrated_device is None: raise SynapseError( HTTPStatus.FORBIDDEN, - "You may only fetch messages for your dehydrated device", + "No dehydrated device exists", Codes.FORBIDDEN, ) - dehydrated_device_id = dehydrated_device[0] + dehydrated_device_id, _ = dehydrated_device if device_id != dehydrated_device_id: raise SynapseError( HTTPStatus.FORBIDDEN, @@ -380,8 +383,7 @@ async def get_events_for_dehydrated_device( ) for message in messages: - # We pop here as we shouldn't be sending the message ID down - # `/sync` + # Remove the message id before sending to client message_id = message.pop("message_id", None) if message_id: set_tag(SynapseTags.TO_DEVICE_EDU_ID, message_id) diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index 89776f639263..44cf76b0af7e 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -32,7 +32,6 @@ from synapse.rest.client.models import AuthenticationData from synapse.rest.models import RequestBodyModel from synapse.types import JsonDict -from synapse.util.cancellation import cancellable if TYPE_CHECKING: from synapse.server import HomeServer @@ -371,7 +370,6 @@ def __init__(self, hs: "HomeServer"): class PostBody(RequestBodyModel): next_batch: Optional[StrictStr] - @cancellable async def on_POST( self, request: SynapseRequest, device_id: str ) -> Tuple[int, JsonDict]: From f7e09335b0e95df27e73059f187695074bc41122 Mon Sep 17 00:00:00 2001 From: "H. Shay" Date: Sun, 23 Jul 2023 16:23:57 -0700 Subject: [PATCH 13/13] apply patch --- synapse/handlers/device.py | 4 +- synapse/handlers/devicemessage.py | 18 +-- synapse/rest/client/devices.py | 178 +++++++++++++++++++++++++++++- tests/handlers/test_device.py | 25 +---- tests/rest/client/test_devices.py | 68 +++++++----- 5 files changed, 233 insertions(+), 60 deletions(-) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 5d12a39e26a4..225fd75bc267 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -653,6 +653,7 @@ async def notify_user_signature_update( async def store_dehydrated_device( self, user_id: str, + device_id: Optional[str], device_data: JsonDict, initial_device_display_name: Optional[str] = None, ) -> str: @@ -661,6 +662,7 @@ async def store_dehydrated_device( Args: user_id: the user that we are storing the device for + device_id: device id supplied by client device_data: the dehydrated device information initial_device_display_name: The display name to use for the device Returns: @@ -668,7 +670,7 @@ async def store_dehydrated_device( """ device_id = await self.check_device_registered( user_id, - None, + device_id, initial_device_display_name, ) old_device_id = await self.store.store_dehydrated_device( diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index 92d4b7b206c4..15e94a03cbe7 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -317,7 +317,7 @@ async def get_events_for_dehydrated_device( ) -> JsonDict: """Fetches up to `limit` events sent to `device_id` starting from `since_token` and returns the new since token. If there are no more messages, returns an empty - array and deletes the dehydrated device associated with the user/device_id. + array. Args: requester: the user requesting the messages @@ -373,7 +373,11 @@ async def get_events_for_dehydrated_device( user_id, device_id, since_stream_id ) logger.debug( - "Deleted %d to-device messages up to %d", deleted, since_stream_id + "Deleted %d to-device messages up to %d for user_id %s device_id %s", + deleted, + since_stream_id, + user_id, + device_id, ) to_token = self.event_sources.get_current_token().to_device_key @@ -389,20 +393,16 @@ async def get_events_for_dehydrated_device( set_tag(SynapseTags.TO_DEVICE_EDU_ID, message_id) logger.debug( - "Returning %d to-device messages between %d and %d (current token: %d) for dehydrated device %s", + "Returning %d to-device messages between %d and %d (current token: %d) for " + "dehydrated device %s, user_id %s", len(messages), since_stream_id, stream_id, to_token, device_id, + user_id, ) - if messages == []: - # we've fetched all the messages, delete the dehydrated device - await self.store.remove_dehydrated_device( - requester.user.to_string(), device_id - ) - return { "events": messages, "next_batch": f"d{stream_id}", diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index 44cf76b0af7e..690d2ec406fc 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -14,12 +14,13 @@ # limitations under the License. import logging +from http import HTTPStatus from typing import TYPE_CHECKING, List, Optional, Tuple from pydantic import Extra, StrictStr from synapse.api import errors -from synapse.api.errors import NotFoundError, UnrecognizedRequestError +from synapse.api.errors import NotFoundError, SynapseError, UnrecognizedRequestError from synapse.handlers.device import DeviceHandler from synapse.http.server import HttpServer from synapse.http.servlet import ( @@ -28,6 +29,7 @@ parse_integer, ) from synapse.http.site import SynapseRequest +from synapse.replication.http.devices import ReplicationUploadKeysForUserRestServlet from synapse.rest.client._base import client_patterns, interactive_auth_handler from synapse.rest.client.models import AuthenticationData from synapse.rest.models import RequestBodyModel @@ -230,7 +232,7 @@ class Config: class DehydratedDeviceServlet(RestServlet): """Retrieve or store a dehydrated device. - Implements either MSC2697 and MSC3814. + Implements either MSC2697 or MSC3814. GET /org.matrix.msc2697.v2/dehydrated_device @@ -301,6 +303,7 @@ async def on_PUT(self, request: SynapseRequest) -> Tuple[int, JsonDict]: device_id = await self.device_handler.store_dehydrated_device( requester.user.to_string(), + None, submission.device_data.dict(), submission.initial_device_display_name, ) @@ -390,6 +393,175 @@ async def on_POST( return 200, msgs +class DehydratedDeviceV2Servlet(RestServlet): + """Upload, retrieve, or delete a dehydrated device. + + GET /org.matrix.msc3814.v1/dehydrated_device + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "device_id": "dehydrated_device_id", + "device_data": { + "algorithm": "org.matrix.msc2697.v1.dehydration.v1.olm", + "account": "dehydrated_device" + } + } + + PUT /org.matrix.msc3814.v1/dehydrated_device + Content-Type: application/json + + { + "device_id": "dehydrated_device_id", + "device_data": { + "algorithm": "org.matrix.msc2697.v1.dehydration.v1.olm", + "account": "dehydrated_device" + }, + "device_keys": { + "user_id": "", + "device_id": "", + "valid_until_ts": , + "algorithms": [ + "m.olm.curve25519-aes-sha2", + ] + "keys": { + ":": "", + }, + "signatures:" { + "" { + ":": "" + } + } + }, + "fallback_keys": { + ":": "", + "signed_:": { + "fallback": true, + "key": "", + "signatures": { + "": { + ":": "" + } + } + } + } + "one_time_keys": { + ":": "" + }, + + } + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "device_id": "dehydrated_device_id" + } + + DELETE /org.matrix.msc3814.v1/dehydrated_device + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "device_id": "dehydrated_device_id", + } + """ + + PATTERNS = [ + *client_patterns("/org.matrix.msc3814.v1/dehydrated_device$", releases=()), + ] + + def __init__(self, hs: "HomeServer"): + super().__init__() + self.hs = hs + self.auth = hs.get_auth() + handler = hs.get_device_handler() + assert isinstance(handler, DeviceHandler) + self.e2e_keys_handler = hs.get_e2e_keys_handler() + self.device_handler = handler + + if hs.config.worker.worker_app is None: + # if main process + self.key_uploader = self.e2e_keys_handler.upload_keys_for_user + else: + # then a worker + self.key_uploader = ReplicationUploadKeysForUserRestServlet.make_client(hs) + + async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: + requester = await self.auth.get_user_by_req(request) + + dehydrated_device = await self.device_handler.get_dehydrated_device( + requester.user.to_string() + ) + + if dehydrated_device is not None: + (device_id, device_data) = dehydrated_device + result = {"device_id": device_id, "device_data": device_data} + return 200, result + else: + raise errors.NotFoundError("No dehydrated device available") + + async def on_DELETE(self, request: SynapseRequest) -> Tuple[int, JsonDict]: + requester = await self.auth.get_user_by_req(request) + + dehydrated_device = await self.device_handler.get_dehydrated_device( + requester.user.to_string() + ) + + if dehydrated_device is not None: + (device_id, device_data) = dehydrated_device + + result = await self.device_handler.rehydrate_device( + requester.user.to_string(), + self.auth.get_access_token_from_request(request), + device_id, + ) + + result = {"device_id": device_id} + + return 200, result + else: + raise errors.NotFoundError("No dehydrated device available") + + class PutBody(RequestBodyModel): + device_data: DehydratedDeviceDataModel + device_id: StrictStr + initial_device_display_name: Optional[StrictStr] + + class Config: + extra = Extra.allow + + async def on_PUT(self, request: SynapseRequest) -> Tuple[int, JsonDict]: + submission = parse_and_validate_json_object_from_request(request, self.PutBody) + requester = await self.auth.get_user_by_req(request) + user_id = requester.user.to_string() + + device_info = submission.dict() + if "device_keys" not in device_info.keys(): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "Device key(s) not found, these must be provided.", + ) + + # TODO: Those two operations, creating a device and storing the + # device's keys should be atomic. + device_id = await self.device_handler.store_dehydrated_device( + requester.user.to_string(), + submission.device_id, + submission.device_data.dict(), + submission.initial_device_display_name, + ) + + # TODO: Do we need to do something with the result here? + await self.key_uploader( + user_id=user_id, device_id=submission.device_id, keys=submission.dict() + ) + + return 200, {"device_id": device_id} + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: if ( hs.config.worker.worker_app is None @@ -404,5 +576,5 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: DehydratedDeviceServlet(hs, msc2697=True).register(http_server) ClaimDehydratedDeviceServlet(hs).register(http_server) if hs.config.experimental.msc3814_enabled: - DehydratedDeviceServlet(hs, msc2697=False).register(http_server) + DehydratedDeviceV2Servlet(hs).register(http_server) DehydratedDeviceEventsServlet(hs).register(http_server) diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index 1be366702b51..56c31217462f 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -430,6 +430,7 @@ def test_dehydrate_and_rehydrate_device(self) -> None: stored_dehydrated_device_id = self.get_success( self.handler.store_dehydrated_device( user_id=user_id, + device_id=None, device_data={"device_data": {"foo": "bar"}}, initial_device_display_name="dehydrated device", ) @@ -506,6 +507,7 @@ def test_dehydrate_v2_and_fetch_events(self) -> None: stored_dehydrated_device_id = self.get_success( self.handler.store_dehydrated_device( user_id=user_id, + device_id=None, device_data={"device_data": {"foo": "bar"}}, initial_device_display_name="dehydrated device", ) @@ -530,7 +532,7 @@ def test_dehydrate_v2_and_fetch_events(self) -> None: requester = create_requester(user_id, device_id=device_id) - # Fetching messages for a non existing device should return an error + # Fetching messages for a non-existing device should return an error self.get_failure( self.message_handler.get_events_for_dehydrated_device( requester=requester, @@ -565,7 +567,8 @@ def test_dehydrate_v2_and_fetch_events(self) -> None: self.assertEqual(len(res["events"]), 1) self.assertEqual(res["events"][0]["content"]["body"], "foo") - # Fetch the message of the dehydrated device again, which should return nothing and delete the old messages + # Fetch the message of the dehydrated device again, which should return nothing + # and delete the old messages res = self.get_success( self.message_handler.get_events_for_dehydrated_device( requester=requester, @@ -576,21 +579,3 @@ def test_dehydrate_v2_and_fetch_events(self) -> None: ) self.assertTrue(len(res["next_batch"]) > 1) self.assertEqual(len(res["events"]), 0) - - # Fetching messages again should fail, since the messages and dehydrated device - # were deleted - self.get_failure( - self.message_handler.get_events_for_dehydrated_device( - requester=requester, - device_id=stored_dehydrated_device_id, - since_token=None, - limit=10, - ), - SynapseError, - ) - - # make sure that the dehydrated device ID is deleted after fetching messages - res2 = self.get_success( - self.handler.get_dehydrated_device(requester.user.to_string()), - ) - self.assertEqual(res2, None) diff --git a/tests/rest/client/test_devices.py b/tests/rest/client/test_devices.py index 25e425b0fe6e..b7d420cfec02 100644 --- a/tests/rest/client/test_devices.py +++ b/tests/rest/client/test_devices.py @@ -233,7 +233,21 @@ def test_PUT(self) -> None: "device_data": { "algorithm": "org.matrix.msc2697.v1.dehydration.v1.olm", "account": "dehydrated_device", - } + }, + "device_keys": { + "user_id": "@alice:test", + "device_id": "device1", + "valid_until_ts": "80", + "algorithms": [ + "m.olm.curve25519-aes-sha2", + ], + "keys": { + ":": "", + }, + "signatures": { + "": {":": ""} + }, + }, }, access_token=token, shorthand=False, @@ -252,44 +266,35 @@ def test_dehydrate_msc3814(self) -> None: "device_data": { "algorithm": "m.dehydration.v1.olm", }, + "device_id": "device1", "initial_device_display_name": "foo bar", - } - channel = self.make_request( - "PUT", - "_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device", - content=content, - access_token=token, - shorthand=False, - ) - self.assertEqual(channel.code, 200) - device_id = channel.json_body.get("device_id") - assert device_id is not None - self.assertIsInstance(device_id, str) - - # test that you can upload keys for this device - content = { "device_keys": { - "algorithms": ["m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"], - "device_id": f"{device_id}", + "user_id": "@mikey:test", + "device_id": "device1", + "valid_until_ts": "80", + "algorithms": [ + "m.olm.curve25519-aes-sha2", + ], "keys": { - "curve25519:JLAFKJWSCS": "3C5BFWi2Y8MaVvjM8M22DBmh24PmgR0nPvJOIArzgyI", - "ed25519:JLAFKJWSCS": "lEuiRJBit0IG6nUf5pUzWTUEsRVVe/HJkoKuEww9ULI", + ":": "", }, "signatures": { - "@alice:example.com": { - "ed25519:JLAFKJWSCS": "dSO80A01XiigH3uBiDVx/EjzaoycHcjq9lfQX0uWsqxl2giMIiSPR8a4d291W1ihKJL/a+myXS367WT6NAIcBA" - } + "": {":": ""} }, - "user_id": f"{user}", }, } channel = self.make_request( - "POST", - f"/_matrix/client/r0/keys/upload/{device_id}", + "PUT", + "_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device", content=content, access_token=token, + shorthand=False, ) self.assertEqual(channel.code, 200) + device_id = channel.json_body.get("device_id") + assert device_id is not None + self.assertIsInstance(device_id, str) + self.assertEqual("device1", device_id) # test that we can now GET the dehydrated device info channel = self.make_request( @@ -358,7 +363,16 @@ def test_dehydrate_msc3814(self) -> None: self.assertEqual(channel.code, 200) self.assertEqual(channel.json_body["events"], []) - # make sure that the dehydrated device id is deleted after we received the messages + # make sure we can delete the dehydrated device + channel = self.make_request( + "DELETE", + "_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device", + access_token=token, + shorthand=False, + ) + self.assertEqual(channel.code, 200) + + # ...and after deleting it is no longer available channel = self.make_request( "GET", "_matrix/client/unstable/org.matrix.msc3814.v1/dehydrated_device",