From e52d94bea9cb7129d545d78e7d684926ddae8c67 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Mon, 25 Jun 2018 15:00:58 +0100 Subject: [PATCH 01/52] ignore malformed PR items --- scripts/proposals.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index b489f0d2..a8f58cae 100755 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -63,7 +63,6 @@ for label in labels: for item in issues[label]: # set the created date, find local field, otherwise Github - print(item) body = str(item['body']) created = re.search('^Date: (.+?)\n', body, flags=re.MULTILINE) if created is not None: @@ -133,16 +132,22 @@ for label in labels: text_file.write(" - " + str(shepherd) + "\n") # PRs - pr_list = re.search('PRs: (.+?)$', str(item['body'])) - if pr_list is not None: - pr_list_formatted = set() - pr_list = pr_list.group(1) - for p in pr_list.split(","): - prs.add(p.strip()) - pr_list_formatted.add("`PR" + str(p.strip()) + "`_") - text_file.write(" - " + ', '.join(pr_list_formatted)) - text_file.write("\n") - else: + try: + pr_list = re.search('PRs: (.+?)$', str(item['body'])) + if pr_list is not None: + pr_list_formatted = set() + pr_list = pr_list.group(1) + for p in pr_list.split(","): + if not re.match(r"#\d", p.strip()): + raise RuntimeWarning + prs.add(p.strip()) + pr_list_formatted.add("`PR" + str(p.strip()) + "`_") + text_file.write(" - " + ', '.join(pr_list_formatted)) + text_file.write("\n") + else: + text_file.write(" - \n") + except: + print("exception parsing PRs for MSC" + str(item['number'])) text_file.write(" - \n") text_file.write("\n\n\n") From a2e0d0382ca46a7564581fed0e3fbfc4093b08fb Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Tue, 3 Jul 2018 14:10:22 -0400 Subject: [PATCH 02/52] ignore emacs temporary files and fix ignoring of dotfiles --- scripts/continuserv/main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index 330ddca7..21effdfd 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -120,8 +120,11 @@ func filter(e fsnotify.Event) bool { return false } - // Avoid some temp files that vim writes - if strings.HasSuffix(e.Name, "~") || strings.HasSuffix(e.Name, ".swp") || strings.HasPrefix(e.Name, ".") { + _, fname := filepath.Split(e.Name) + + // Avoid some temp files that vim or emacs writes + if strings.HasSuffix(e.Name, "~") || strings.HasSuffix(e.Name, ".swp") || strings.HasPrefix(fname, ".") || + (strings.HasPrefix(fname, "#") && strings.HasSuffix(fname, "#")) { return false } From f09c4fd286e344512ae1890650c5d7bcb0bb022c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 21:35:12 -0600 Subject: [PATCH 03/52] Convert joins to swagger --- api/server-server/joins.yaml | 275 ++++++++++++++++++++++++++++ specification/server_server_api.rst | 11 +- 2 files changed, 276 insertions(+), 10 deletions(-) create mode 100644 api/server-server/joins.yaml diff --git a/api/server-server/joins.yaml b/api/server-server/joins.yaml new file mode 100644 index 00000000..d4a7ed9e --- /dev/null +++ b/api/server-server/joins.yaml @@ -0,0 +1,275 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Join Room API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/make_join/{roomId}/{userId}": + get: + summary: Get information required to make a join event for a room + description: |- + Asks the receiving server to return information that the sending + server will need to prepare a join event to get into the room. + This is part of the `Joining Rooms`_ handshake. + operationId: makeJoin + parameters: + - in: path + name: roomId + type: string + description: The room ID that is about to be joined + required: true + x-example: "!abc123:matrix.org" + - in: path + name: userId + type: string + description: The user ID the join event will be for + required: true + x-example: "@someone:example.org" + responses: + 200: + description: |- + An unsigned event that the server may now use as a template + for the rest of the `Joining Rooms`_ handshake. + schema: + allOf: + - $ref: "definitions/unsigned_pdu.yaml" + - type: object + properties: + # Note: we override a bunch of parameters to change their descriptions + sender: + type: string + description: The user ID of the joining member + example: "@someone:example.org" + origin: + type: string + description: The name of the resident homeserver + example: "matrix.org" + origin_server_ts: + type: integer + format: int64 + description: A timestamp added by the resident homeserver + example: 1234567890 + type: + type: string + description: The value ``m.room.member`` + required: true + example: "m.room.member" + state_key: + type: string + description: The user ID of the joining member + example: "@someone:example.org" + content: + type: object + title: Membership Event Content + description: The content of the event + example: {"membership": "join"} + properties: + membership: + type: string + description: The value ``join`` + example: "join" + required: ['membership'] + depth: + type: integer + description: This field must be present but is ignored; it may be 0 + example: 12 + auth_events: + type: array + description: |- + An event reference list containing the authorization events that would + allow the member to join the room + items: + type: array + maxItems: 2 + minItems: 2 + items: + - type: string + title: Event ID + example: "$abc123:matrix.org" + - type: object + title: Event Hash + example: { + "sha256": "abase64encodedsha256hashshouldbe43byteslong" + } + properties: + sha256: + type: string + description: The event hash + example: abase64encodedsha256hashshouldbe43byteslong + required: ['sha256'] + redacts: + type: string + description: Not used + required: + # Every other field is already flagged as required by the $ref + - state_key + examples: + application/json: { + "$ref": "examples/unsigned_pdu.json", + "type": "m.room.member", + "content": { + "membership": "join" + } + } + "/send_join/{roomId}/{eventId}": + put: + summary: Submit a signed join event to a resident server + description: |- + Submits a signed join event to the resident server for it + to accept it into the room's graph. + operationId: sendJoin + parameters: + - in: path + name: roomId + type: string + description: The room ID that is about to be joined + required: true + x-example: "!abc123:matrix.org" + - in: path + name: eventId + type: string + description: The event ID for the join event + required: true + x-example: "$abc123:example.org" + - in: body + name: body + type: object + required: true + schema: + allOf: + - $ref: "definitions/pdu.yaml" + - type: object + properties: + # Note: we override a bunch of parameters to change their descriptions + sender: + type: string + description: The user ID of the joining member + example: "@someone:example.org" + origin: + type: string + description: The name of the joining homeserver + example: "matrix.org" + origin_server_ts: + type: integer + format: int64 + description: A timestamp added by the joining homeserver + example: 1234567890 + type: + type: string + description: The value ``m.room.member`` + example: "m.room.member" + state_key: + type: string + description: The user ID of the joining member + example: "@someone:example.org" + content: + type: object + title: Membership Event Content + description: The content of the event + example: {"membership": "join"} + properties: + membership: + type: string + description: The value ``join`` + example: "join" + required: ['membership'] + depth: + type: integer + description: This field must be present but is ignored; it may be 0 + example: 12 + auth_events: + type: array + description: |- + An event reference list containing the authorization events that would + allow the member to join the room + items: + type: array + maxItems: 2 + minItems: 2 + items: + - type: string + title: Event ID + example: "$abc123:matrix.org" + - type: object + title: Event Hash + example: { + "sha256": "abase64encodedsha256hashshouldbe43byteslong" + } + properties: + sha256: + type: string + description: The event hash + example: abase64encodedsha256hashshouldbe43byteslong + required: ['sha256'] + redacts: + type: string + description: Not used + required: + # Every other field is already flagged as required by the $ref + - state_key + example: { + "$ref": "examples/pdu.json", + "type": "m.room.member", + "content": { + "membership": "join" + } + } + responses: + 200: + description: |- + The full state for the room, having accepted the join event + schema: + type: array + minItems: 2 + maxItems: 2 + items: + - type: integer + description: The value ``200`` + example: 200 + - type: object + title: Room State + description: The state for the room + properties: + auth_chain: + type: array + description: The auth chain + items: + type: object + properties: {} + # TODO: Verify schema + state: + type: array + description: The room state + items: + type: object + properties: {} + # TODO: Verify schema + required: ["auth_chain", "state"] + examples: + application/json: [ + 200, + { + # TODO: Use the appropriate refs (see TODOs in schema) + "auth_chain": [], + "state": [] + } + ] diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 6f980045..e8960d11 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -689,16 +689,7 @@ All these URLs are name-spaced within a prefix of:: {{query_general_ss_http_api}} -To join a room:: - - GET .../make_join// - Response: JSON encoding of a join proto-event - - PUT .../send_join// - Response: JSON encoding of the state of the room at the time of the event - -Performs the room join handshake. For more information, see "Joining Rooms" -below. +{{joins_ss_http_api}} Joining Rooms ------------- From 004998b98f6ec2e1b15c49e3c27674d92ddf6a28 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 22:10:46 -0600 Subject: [PATCH 04/52] Convert invites to swagger The whole section reads like a description for the endpoint, and has been replaced by the swagger definition now (rather than at a later stage). All the same information should be kept. --- .../definitions/invite_event.yaml | 88 +++++++++++++++++++ api/server-server/invites.yaml | 84 ++++++++++++++++++ specification/server_server_api.rst | 60 +------------ 3 files changed, 173 insertions(+), 59 deletions(-) create mode 100644 api/server-server/definitions/invite_event.yaml create mode 100644 api/server-server/invites.yaml diff --git a/api/server-server/definitions/invite_event.yaml b/api/server-server/definitions/invite_event.yaml new file mode 100644 index 00000000..989c159f --- /dev/null +++ b/api/server-server/definitions/invite_event.yaml @@ -0,0 +1,88 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +title: Invite Event +description: An invite event +allOf: + - $ref: "pdu.yaml" + - type: object + properties: + # Note: we override a bunch of parameters to change their descriptions + sender: + type: string + # TODO: Verify/clarify this - it doesn't seem right, given this is a 'regular' invite + description: |- + The matrix ID of the user who sent the original ``m.room.third_party_invite`` + example: "@someone:example.org" + origin: + type: string + description: The name of the inviting homeserver + example: "matrix.org" + origin_server_ts: + type: integer + format: int64 + description: A timestamp added by the inviting homeserver + example: 1234567890 + type: + type: string + description: The value ``m.room.member`` + example: "m.room.member" + state_key: + type: string + description: The user ID of the invited member + example: "@joe:elsewhere.com" + content: + type: object + title: Membership Event Content + description: |- + The content of the event, matching what is available in the + `Client-Server API`_ + example: {"membership": "invite"} + properties: + membership: + type: string + description: The value ``invite`` + example: "invite" + required: ['membership'] + auth_events: + type: array + description: |- + An event reference list containing the authorization events that would + allow the member to be invited to the room + items: + type: array + maxItems: 2 + minItems: 2 + items: + - type: string + title: Event ID + example: "$abc123:matrix.org" + - type: object + title: Event Hash + example: { + "sha256": "abase64encodedsha256hashshouldbe43byteslong" + } + properties: + sha256: + type: string + description: The event hash + example: abase64encodedsha256hashshouldbe43byteslong + required: ['sha256'] + redacts: + type: string + description: Not used + required: + # Every other field is already flagged as required by the $ref + - state_key + - unsigned # TODO: apparently this is required? diff --git a/api/server-server/invites.yaml b/api/server-server/invites.yaml new file mode 100644 index 00000000..ad20241b --- /dev/null +++ b/api/server-server/invites.yaml @@ -0,0 +1,84 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Invite User To Room API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/invite/{roomId}/{eventId}": + put: + summary: Invites a user to a room + description: |- + Invites a remote user to a room. Once the event has been + signed by both the inviting homeserver and the invited + homeserver, it can be sent to all of the users in the room. + operationId: sendInvite + parameters: + - in: path + name: roomId + type: string + description: The room ID that the user is being invited to + required: true + x-example: "!abc123:matrix.org" + - in: path + name: eventId + type: string + description: The event ID for the invite event + required: true + x-example: "$abc123:example.org" + - in: body + name: body + type: object + required: true + schema: + $ref: "definitions/invite_event.yaml" + example: { + "$ref": "examples/pdu.json", + "type": "m.room.member", + "content": { + "membership": "invite" + } + } + responses: + 200: + description: |- + The event with the invited server's signature added. All other fields of the events + should remain untouched. + schema: + type: array + minItems: 2 + maxItems: 2 + items: + - type: integer + description: The value ``200`` + example: 200 + - $ref: "definitions/invite_event.yaml" + examples: + application/json: [ + 200, + { + "$ref": "examples/pdu.json", + "type": "m.room.member", + "content": { + "membership": "invite" + } + } + ] diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index e8960d11..f1825f27 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -879,65 +879,7 @@ that requested by the requester in the ``v`` parameter). Inviting to a room ------------------ -When a user wishes to invite another user to a local room and the other user is -on a different server, the inviting server will send a request to the invited -server:: - - PUT .../invite/{roomId}/{eventId} - -The required fields in the JSON body are: - -======================== ============ ========================================= - Key Type Description -======================== ============ ========================================= -``room_id`` String The room ID of the room. Must be the same - as the room ID specified in the path. -``event_id`` String The ID of the event. Must be the same as - the event ID specified in the path. -``type`` String The value ``m.room.member``. -``auth_events`` List An event-reference list containing the - IDs of the authorization events that - would allow this member to be invited in - the room. -``content`` Object The content of the event. -``depth`` Integer The depth of the event. -``origin`` String The name of the inviting homeserver. -``origin_server_ts`` Integer A timestamp added by the inviting - homeserver. -``prev_events`` List An event-reference list containing the - IDs of the immediate predecessor events. -``sender`` String The Matrix ID of the user who sent the - original ``m.room.third_party_invite``. -``state_key`` String The Matrix ID of the invited user. -``signatures`` Object The signature of the event from the - origin server. -``unsigned`` Object An object containing the properties that - aren't part of the signature's - computation. -======================== ============ ========================================= - -Where the ``content`` key contains the content for the ``m.room.member`` event -specified in the `Client-Server API`_. Note that the ``membership`` property of -the content must be ``invite``. - -Upon receiving this request, the invited homeserver will append its signature to -the event and respond to the request with the following JSON body:: - - [ - 200, - "event": {...} - ] - -Where ``event`` contains the event signed by both homeservers, using the same -JSON keys as the initial request on ``/invite/{roomId}/{eventId}``. Note that, -except for the ``signatures`` object (which now contains an additional signature), -all of the event's keys remain the same as in the event initially provided. - -This response format is due to a typo in Synapse, the first implementation of -Matrix's APIs, and is preserved to maintain compatibility. - -Now that the event has been signed by both the inviting homeserver and the -invited homeserver, it can be sent to all of the users in the room. +{{invites_ss_http_api}} Third-party invites ------------------- From 9c1c541a3829642b3371d8523d0fd3f13c095e8e Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 13 Jul 2018 15:04:22 -0600 Subject: [PATCH 05/52] Remove extra "required" declarations "required" is something that is handled as an array, and these are invalid. The `type` in both cases is covered by a required array elsewhere --- api/server-server/joins.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/api/server-server/joins.yaml b/api/server-server/joins.yaml index d4a7ed9e..09714724 100644 --- a/api/server-server/joins.yaml +++ b/api/server-server/joins.yaml @@ -71,7 +71,6 @@ paths: type: type: string description: The value ``m.room.member`` - required: true example: "m.room.member" state_key: type: string From 45ec8269e68c3c0f033f5414ff3e45e763ff7236 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 13 Jul 2018 15:04:45 -0600 Subject: [PATCH 06/52] Fill out the required fields in all the examples --- api/server-server/examples/unsigned_pdu.json | 7 +++++++ api/server-server/invites.yaml | 8 ++++++-- api/server-server/joins.yaml | 2 ++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/api/server-server/examples/unsigned_pdu.json b/api/server-server/examples/unsigned_pdu.json index 6ed4b79d..0b585a63 100644 --- a/api/server-server/examples/unsigned_pdu.json +++ b/api/server-server/examples/unsigned_pdu.json @@ -4,6 +4,13 @@ "origin": "example.com", "event_id": "$a4ecee13e2accdadf56c1025:example.com", "origin_server_ts": 1404838188000, + "depth": 12, + "auth_events": [ + [ + "$af232176:example.org", + {"sha256": "abase64encodedsha256hashshouldbe43byteslong"} + ] + ], "type": "m.room.message", "prev_events": [ [ diff --git a/api/server-server/invites.yaml b/api/server-server/invites.yaml index ad20241b..d176a5ae 100644 --- a/api/server-server/invites.yaml +++ b/api/server-server/invites.yaml @@ -53,9 +53,11 @@ paths: example: { "$ref": "examples/pdu.json", "type": "m.room.member", + "state_key": "@someone:example.org", "content": { "membership": "invite" - } + }, + "unsigned": {} } responses: 200: @@ -77,8 +79,10 @@ paths: { "$ref": "examples/pdu.json", "type": "m.room.member", + "state_key": "@someone:example.org", "content": { "membership": "invite" - } + }, + "unsigned": {} } ] diff --git a/api/server-server/joins.yaml b/api/server-server/joins.yaml index 09714724..b3aebf14 100644 --- a/api/server-server/joins.yaml +++ b/api/server-server/joins.yaml @@ -125,6 +125,7 @@ paths: application/json: { "$ref": "examples/unsigned_pdu.json", "type": "m.room.member", + "state_key": "@someone:example.org", "content": { "membership": "join" } @@ -228,6 +229,7 @@ paths: example: { "$ref": "examples/pdu.json", "type": "m.room.member", + "state_key": "@someone:example.org", "content": { "membership": "join" } From 7314405904b507290d74188c783e8f3b6db7c293 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 13 Jul 2018 15:04:52 -0600 Subject: [PATCH 07/52] Update comment --- api/server-server/definitions/invite_event.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server-server/definitions/invite_event.yaml b/api/server-server/definitions/invite_event.yaml index 989c159f..d87a1daa 100644 --- a/api/server-server/definitions/invite_event.yaml +++ b/api/server-server/definitions/invite_event.yaml @@ -85,4 +85,4 @@ allOf: required: # Every other field is already flagged as required by the $ref - state_key - - unsigned # TODO: apparently this is required? + - unsigned # TODO: apparently this is required? Verify. From 797aca87bd28c3327a4b855f9dd572cfe02b4373 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 13 Jul 2018 15:05:44 -0600 Subject: [PATCH 08/52] Update the check_examples.py script to resolve references The script previously wasn't chasing down valid references in examples. --- api/check_examples.py | 45 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/api/check_examples.py b/api/check_examples.py index be0676bb..fb9430b6 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -56,7 +56,8 @@ def check_parameter(filepath, request, parameter): # Setting the 'id' tells jsonschema where the file is so that it # can correctly resolve relative $ref references in the schema schema['id'] = fileurl - resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_yaml}) + example = resolve_references(filepath, example) + resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file}) jsonschema.validate(example, schema, resolver=resolver) except Exception as e: raise ValueError("Error validating JSON schema for %r" % ( @@ -76,7 +77,8 @@ def check_response(filepath, request, code, response): # Setting the 'id' tells jsonschema where the file is so that it # can correctly resolve relative $ref references in the schema schema['id'] = fileurl - resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_yaml}) + example = resolve_references(filepath, example) + resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file}) jsonschema.validate(example, schema, resolver=resolver) except Exception as e: raise ValueError("Error validating JSON schema for %r %r" % ( @@ -104,6 +106,27 @@ def check_swagger_file(filepath): check_response(filepath, request, code, response) +def resolve_references(path, schema): + if isinstance(schema, dict): + # do $ref first + if '$ref' in schema: + value = schema['$ref'] + path = os.path.abspath(os.path.join(os.path.dirname(path), value)) + ref = load_file("file://" + path) + result = resolve_references(path, ref) + del schema['$ref'] + else: + result = {} + + for key, value in schema.items(): + result[key] = resolve_references(path, value) + return result + elif isinstance(schema, list): + return [resolve_references(path, value) for value in schema] + else: + return schema + + def load_yaml(path): if not path.startswith("file:///"): raise Exception("Bad ref: %s" % (path,)) @@ -112,6 +135,24 @@ def load_yaml(path): return yaml.load(f) +def load_json(path): + if not path.startswith("file:///"): + raise Exception("Bad ref: %s" % (path,)) + path = path[len("file://"):] + with open(path, "r") as f: + return json.load(f) + + +def load_file(path): + print("Loading reference: %s" % path) + if path.endswith(".json"): + return load_json(path) + else: + # We have to assume it's YAML because some of the YAML examples + # do not have file extensions. + return load_yaml(path) + + if __name__ == '__main__': paths = sys.argv[1:] if not paths: From eaf175056f48ca137dbde4c5d27f771b680b3c78 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 18 Jul 2018 10:59:04 -0600 Subject: [PATCH 09/52] Full stops --- .../definitions/invite_event.yaml | 20 +++---- api/server-server/invites.yaml | 6 +- api/server-server/joins.yaml | 60 +++++++++---------- 3 files changed, 43 insertions(+), 43 deletions(-) diff --git a/api/server-server/definitions/invite_event.yaml b/api/server-server/definitions/invite_event.yaml index d87a1daa..9cef8061 100644 --- a/api/server-server/definitions/invite_event.yaml +++ b/api/server-server/definitions/invite_event.yaml @@ -23,43 +23,43 @@ allOf: type: string # TODO: Verify/clarify this - it doesn't seem right, given this is a 'regular' invite description: |- - The matrix ID of the user who sent the original ``m.room.third_party_invite`` + The matrix ID of the user who sent the original ``m.room.third_party_invite``. example: "@someone:example.org" origin: type: string - description: The name of the inviting homeserver + description: The name of the inviting homeserver. example: "matrix.org" origin_server_ts: type: integer format: int64 - description: A timestamp added by the inviting homeserver + description: A timestamp added by the inviting homeserver. example: 1234567890 type: type: string - description: The value ``m.room.member`` + description: The value ``m.room.member``. example: "m.room.member" state_key: type: string - description: The user ID of the invited member + description: The user ID of the invited member. example: "@joe:elsewhere.com" content: type: object title: Membership Event Content description: |- The content of the event, matching what is available in the - `Client-Server API`_ + `Client-Server API`_. example: {"membership": "invite"} properties: membership: type: string - description: The value ``invite`` + description: The value ``invite``. example: "invite" required: ['membership'] auth_events: type: array description: |- An event reference list containing the authorization events that would - allow the member to be invited to the room + allow the member to be invited to the room. items: type: array maxItems: 2 @@ -76,12 +76,12 @@ allOf: properties: sha256: type: string - description: The event hash + description: The event hash. example: abase64encodedsha256hashshouldbe43byteslong required: ['sha256'] redacts: type: string - description: Not used + description: Not used. required: # Every other field is already flagged as required by the $ref - state_key diff --git a/api/server-server/invites.yaml b/api/server-server/invites.yaml index d176a5ae..a5d21938 100644 --- a/api/server-server/invites.yaml +++ b/api/server-server/invites.yaml @@ -35,13 +35,13 @@ paths: - in: path name: roomId type: string - description: The room ID that the user is being invited to + description: The room ID that the user is being invited to. required: true x-example: "!abc123:matrix.org" - in: path name: eventId type: string - description: The event ID for the invite event + description: The event ID for the invite event. required: true x-example: "$abc123:example.org" - in: body @@ -70,7 +70,7 @@ paths: maxItems: 2 items: - type: integer - description: The value ``200`` + description: The value ``200``. example: 200 - $ref: "definitions/invite_event.yaml" examples: diff --git a/api/server-server/joins.yaml b/api/server-server/joins.yaml index b3aebf14..11a47af6 100644 --- a/api/server-server/joins.yaml +++ b/api/server-server/joins.yaml @@ -35,13 +35,13 @@ paths: - in: path name: roomId type: string - description: The room ID that is about to be joined + description: The room ID that is about to be joined. required: true x-example: "!abc123:matrix.org" - in: path name: userId type: string - description: The user ID the join event will be for + description: The user ID the join event will be for. required: true x-example: "@someone:example.org" responses: @@ -57,29 +57,29 @@ paths: # Note: we override a bunch of parameters to change their descriptions sender: type: string - description: The user ID of the joining member + description: The user ID of the joining member. example: "@someone:example.org" origin: type: string - description: The name of the resident homeserver + description: The name of the resident homeserver. example: "matrix.org" origin_server_ts: type: integer format: int64 - description: A timestamp added by the resident homeserver + description: A timestamp added by the resident homeserver. example: 1234567890 type: type: string - description: The value ``m.room.member`` + description: The value ``m.room.member``. example: "m.room.member" state_key: type: string - description: The user ID of the joining member + description: The user ID of the joining member. example: "@someone:example.org" content: type: object title: Membership Event Content - description: The content of the event + description: The content of the event. example: {"membership": "join"} properties: membership: @@ -89,13 +89,13 @@ paths: required: ['membership'] depth: type: integer - description: This field must be present but is ignored; it may be 0 + description: This field must be present but is ignored; it may be 0. example: 12 auth_events: type: array description: |- An event reference list containing the authorization events that would - allow the member to join the room + allow the member to join the room. items: type: array maxItems: 2 @@ -112,12 +112,12 @@ paths: properties: sha256: type: string - description: The event hash + description: The event hash. example: abase64encodedsha256hashshouldbe43byteslong required: ['sha256'] redacts: type: string - description: Not used + description: Not used. required: # Every other field is already flagged as required by the $ref - state_key @@ -141,13 +141,13 @@ paths: - in: path name: roomId type: string - description: The room ID that is about to be joined + description: The room ID that is about to be joined. required: true x-example: "!abc123:matrix.org" - in: path name: eventId type: string - description: The event ID for the join event + description: The event ID for the join event. required: true x-example: "$abc123:example.org" - in: body @@ -162,45 +162,45 @@ paths: # Note: we override a bunch of parameters to change their descriptions sender: type: string - description: The user ID of the joining member + description: The user ID of the joining member. example: "@someone:example.org" origin: type: string - description: The name of the joining homeserver + description: The name of the joining homeserver. example: "matrix.org" origin_server_ts: type: integer format: int64 - description: A timestamp added by the joining homeserver + description: A timestamp added by the joining homeserver. example: 1234567890 type: type: string - description: The value ``m.room.member`` + description: The value ``m.room.member``. example: "m.room.member" state_key: type: string - description: The user ID of the joining member + description: The user ID of the joining member. example: "@someone:example.org" content: type: object title: Membership Event Content - description: The content of the event + description: The content of the event. example: {"membership": "join"} properties: membership: type: string - description: The value ``join`` + description: The value ``join``. example: "join" required: ['membership'] depth: type: integer - description: This field must be present but is ignored; it may be 0 + description: This field must be present but is ignored; it may be 0. example: 12 auth_events: type: array description: |- An event reference list containing the authorization events that would - allow the member to join the room + allow the member to join the room. items: type: array maxItems: 2 @@ -217,12 +217,12 @@ paths: properties: sha256: type: string - description: The event hash + description: The event hash. example: abase64encodedsha256hashshouldbe43byteslong required: ['sha256'] redacts: type: string - description: Not used + description: Not used. required: # Every other field is already flagged as required by the $ref - state_key @@ -237,29 +237,29 @@ paths: responses: 200: description: |- - The full state for the room, having accepted the join event + The full state for the room, having accepted the join event. schema: type: array minItems: 2 maxItems: 2 items: - type: integer - description: The value ``200`` + description: The value ``200``. example: 200 - type: object title: Room State - description: The state for the room + description: The state for the room. properties: auth_chain: type: array - description: The auth chain + description: The auth chain. items: type: object properties: {} # TODO: Verify schema state: type: array - description: The room state + description: The room state. items: type: object properties: {} From 0d1b82a6017a1e3c0812a3e06d6bd18026a0e96b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 14:40:32 -0600 Subject: [PATCH 10/52] Add the event_id to unsigned PDUs This went missing in the swagger conversion. The example already has an event_id. --- api/server-server/definitions/unsigned_pdu.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml index 011b2736..9e54a837 100644 --- a/api/server-server/definitions/unsigned_pdu.yaml +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -17,6 +17,10 @@ description: An unsigned persistent data unit (event) example: $ref: "../examples/unsigned_pdu.json" properties: + event_id: + type: string + description: The event ID for the PDU + example: "$a4ecee13e2accdadf56c1025:example.com" room_id: type: string description: Room identifier. @@ -107,6 +111,7 @@ properties: description: Additional data added by the origin server but not covered by the ``signatures``. example: {"key": "value"} required: + - event_id - room_id - sender - origin From 9f399a08fe7b7572844704635902e2759405c554 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 14:41:42 -0600 Subject: [PATCH 11/52] Full stops I'm never going to remember these --- api/server-server/definitions/unsigned_pdu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml index 9e54a837..0aeede51 100644 --- a/api/server-server/definitions/unsigned_pdu.yaml +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -19,7 +19,7 @@ example: properties: event_id: type: string - description: The event ID for the PDU + description: The event ID for the PDU. example: "$a4ecee13e2accdadf56c1025:example.com" room_id: type: string From dcc4b0bfbc63f21ce921b4cfbc89133dedbce435 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 14:47:47 -0600 Subject: [PATCH 12/52] Remove Travis CI configuration We're using Circle CI for testing a bunch of the spec, and so far it has the advantage of being faster. Also, it's not named the same as me. We can also remove the Travis CI specific check in generate-matrix-org-assets because Circle CI doesn't touch this. --- .travis.yml | 16 ---------------- scripts/generate-matrix-org-assets | 7 ++----- 2 files changed, 2 insertions(+), 21 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9e9363af..00000000 --- a/.travis.yml +++ /dev/null @@ -1,16 +0,0 @@ -language: go -go: - - 1.8 - -sudo: required - -# we only need a single git commit -git: - depth: 1 - -install: - - sudo apt-get update - - sudo apt-get install python3 python3-dev - -script: - - ./scripts/test-and-build.sh diff --git a/scripts/generate-matrix-org-assets b/scripts/generate-matrix-org-assets index cbebbb64..ed08f81d 100755 --- a/scripts/generate-matrix-org-assets +++ b/scripts/generate-matrix-org-assets @@ -8,11 +8,8 @@ cd `dirname $0`/.. mkdir -p assets -if [ "$TRAVIS" != "true" ] -then - # generate specification/proposals.rst - ./scripts/proposals.py -fi +# generate specification/proposals.rst +./scripts/proposals.py # generate the spec docs ./scripts/gendoc.py -d assets/spec From af0629a5204a5a7188126d97783a6316780399a9 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 08:36:12 -0600 Subject: [PATCH 13/52] Full stop --- api/server-server/joins.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server-server/joins.yaml b/api/server-server/joins.yaml index 11a47af6..eaf14e71 100644 --- a/api/server-server/joins.yaml +++ b/api/server-server/joins.yaml @@ -84,7 +84,7 @@ paths: properties: membership: type: string - description: The value ``join`` + description: The value ``join``. example: "join" required: ['membership'] depth: From 0ddf578b6159e0c8effa5cbbb019efc9290fdff9 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 20 Jul 2018 11:26:16 -0600 Subject: [PATCH 14/52] Combine all queries into their own section This removes the Directory and Profile sections, instead opting to document them as Queries. The behaviour of profile queries is based on Synapse's behaviour. A few issues have been opened to improve the behaviour: * https://github.com/matrix-org/matrix-doc/issues/1434 * https://github.com/matrix-org/matrix-doc/issues/1435 * https://github.com/matrix-org/matrix-doc/issues/1436 * https://github.com/matrix-org/matrix-doc/issues/1437 This fixes https://github.com/matrix-org/matrix-doc/issues/1404 --- api/server-server/directory.yaml | 69 ---------- api/server-server/query.yaml | 181 +++++++++++++++++++++++++++ api/server-server/query_general.yaml | 44 ------- specification/server_server_api.rst | 38 ++---- 4 files changed, 190 insertions(+), 142 deletions(-) delete mode 100644 api/server-server/directory.yaml create mode 100644 api/server-server/query.yaml delete mode 100644 api/server-server/query_general.yaml diff --git a/api/server-server/directory.yaml b/api/server-server/directory.yaml deleted file mode 100644 index 94f115b9..00000000 --- a/api/server-server/directory.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2017 Kamax.io -# Copyright 2018 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -swagger: '2.0' -info: - title: "Matrix Federation Query API" - version: "1.0.0" -host: localhost:8448 -schemes: - - https -basePath: /_matrix/federation/v1 -produces: - - application/json -paths: - "/query/directory": - get: - summary: Retrieve the room ID and list of resident homeservers for a room - alias. - description: Retrieve the room ID and list of resident homeservers for a room - alias. - parameters: - - in: query - name: room_alias - type: string - description: Room alias. - required: true - x-example: "#room_alias:example.org" - responses: - 200: - description: The corresponding room ID and list of known resident - homeservers for the room. - schema: - type: object - properties: - room_id: - type: string - description: The room ID mapped to the queried room alias. - x-example: "!roomid1234:example.org" - servers: - type: array - description: An array of server names that are likely to hold - then given room. This list may or may not include the server - answering the query. - items: - type: string - required: - - "room_id" - - "servers" - examples: - application/json: { - "room_id": "!roomid1234:example.org", - "servers": [ - "example.org", - "example.com", - "another.example.com:8449", - ] - } diff --git a/api/server-server/query.yaml b/api/server-server/query.yaml new file mode 100644 index 00000000..23b9a575 --- /dev/null +++ b/api/server-server/query.yaml @@ -0,0 +1,181 @@ +# Copyright 2017 Kamax.io +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Query API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/query/{queryType}": + get: + summary: Query for information + description: |- + Performs a single query request on the receiving homeserver. The query string + arguments are dependent on which type of query is being made. Known query types + are specified as their own endpoints as an extension to this definition. + operationId: queryInfo + parameters: + - in: path + name: queryType + type: string + description: The type of query to make + required: true + x-example: profile + responses: + 200: + description: |- + The query response. The schema varies depending on the query being made. + "/query/directory": + get: + summary: Query for the room ID and resident homeservers for a room alias + description: |- + Performs a query to get the mapped room ID and list of resident homeservers in + the room for a given room alias. Homeservers should only query room aliases + that belong to the target server (idenfified by the DNS Name in the alias). + The target server may not appear in the resident servers list. + + Servers may wish to cache the response to this query to prevent requesting the + information too often. + operationId: queryRoomDirectory + parameters: + - in: query + name: room_alias + type: string + description: The room alias to query. + required: true + x-example: "#room_alias:example.org" + responses: + 200: + description: |- + The corresponding room ID and list of known resident homeservers for the room. + schema: + type: object + properties: + room_id: + type: string + description: The room ID mapped to the queried room alias. + x-example: "!roomid1234:example.org" + servers: + type: array + description: |- + An array of server names that are likely to hold then given room. This + list may or may not include the server answering the query. + items: + type: string + required: + - "room_id" + - "servers" + examples: + application/json: { + "room_id": "!roomid1234:example.org", + "servers": [ + "example.org", + "example.com", + "another.example.com:8449", + ] + } + 400: + description: |- + The room alias is not hosted on the server. This can happen if the directory + server is named "example.org" and the room alias ends with "matrix.org". + schema: + $ref: "../client-server/definitions/errors/error.yaml" + examples: + application/json: { + "errcode": "M_UNKNOWN", + "error": "Room alias not hosted on this homeserver." + } + 404: + description: The room alias was not found. + schema: + $ref: "../client-server/definitions/errors/error.yaml" + examples: + application/json: { + "errcode": "M_NOT_FOUND", + "error": "Room alias not found." + } + "/query/profile": + get: + summary: Query for profile information about a given user + description: |- + Performs a query to get profile information, such as a display name or avatar, + for a given user. Homeservers should only query profiles for users that belong + to the target server (identified by the DNS Name in the user ID). + + Servers may wish to cache the response to this query to prevent requesting the + information too often. + parameters: + - in: query + name: user_id + type: string + description: The user ID to query. + required: true + x-example: "@someone:example.org" + - in: query + name: field + type: string + description: |- + The field to query. If specified, the server will only return the given field + in the response. If not specified, the server will return the full profile for + the user. + responses: + 200: + description: |- + The profile for the user. If a ``field`` is specified in the request, only the + matching field should be included in the response. If no ``field`` was specified, + the response should include the fields of the user's profile that can be made + public, such as the display name and avatar. + + If the ``field`` is for a field that the server does not recognize, an empty object + should be returned. If the ``field`` is recognized, but the user does not have it + set on their profile, ``null`` should be returned for the value of that field. + + If the user does not exist, an empty object should be returned regardless of the + ``field`` being queried. + schema: + type: object + properties: + displayname: + type: string + description: |- + The display name of the user. If the user does not have a display name set, + this should be the value ``null``. + x-example: "John Doe" + avatar_url: + type: string + description: |- + The avatar URL for the user's avatar. If the user does not have an avatar + set, this should be the value ``null``. + x-example: "mxc://matrix.org/MyC00lAvatar" + examples: + application/json: { + "displayname": "John Doe", + "avatar_url": "mxc://matrix.org/MyC00lAvatar" + } + 400: + description: The user does not belong to the server. + schema: + $ref: "../client-server/definitions/errors/error.yaml" + examples: + application/json: { + "errcode": "M_UNKNOWN", + "error": "User is not hosted on this homeserver." + } diff --git a/api/server-server/query_general.yaml b/api/server-server/query_general.yaml deleted file mode 100644 index 97e3406e..00000000 --- a/api/server-server/query_general.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2018 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -swagger: '2.0' -info: - title: "Matrix Federation Query API" - version: "1.0.0" -host: localhost:8448 -schemes: - - https -basePath: /_matrix/federation/v1 -produces: - - application/json -paths: - "/query/{queryType}": - get: - summary: Query for information - description: |- - Performs a single query request on the receiving homeserver. The query string - arguments are dependent on which type of query is being made. Known query types - are specified as their own endpoints as an extension to this definition. - operationId: queryInfo - parameters: - - in: path - name: queryType - type: string - description: The type of query to make - required: true - x-example: profile - responses: - 200: - description: |- - The query response. The schema varies depending on the query being made. diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f1825f27..b7d5a82f 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -686,8 +686,6 @@ All these URLs are name-spaced within a prefix of:: {{events_ss_http_api}} -{{query_general_ss_http_api}} - {{joins_ss_http_api}} @@ -1107,36 +1105,18 @@ Rejecting a presence invite:: - Explain the zero-byte presence inference logic See also: docs/client-server/model/presence -Profiles --------- +Querying for information +------------------------ -The server API for profiles is based entirely on the following Federation -Queries. There are no additional EDU or PDU types involved, other than the -implicit ``m.presence`` and ``m.room.member`` events (see section below). +Queries are a way to retrieve information from a homeserver abotu a resource, +such as a user or room. The endpoints here are often called in conjunction with +a request from a client on the client-server API in order to complete the call. -Querying profile information:: +There are several types of queries that can be made. The generic endpoint to +represent all queries is described first, followed by the more specific queries +that can be made. - Query type: profile - - Arguments: - user_id: the ID of the user whose profile to return - field: (optional) string giving a field name - - Returns: JSON object containing the following keys: - displayname: string of free-form text - avatar_url: string containing an HTTP-scheme URL - -If the query contains the optional ``field`` key, it should give the name of a -result field. If such is present, then the result should contain only a field -of that name, with no others present. If not, the result should contain as much -of the user's profile as the homeserver has available and can make public. - -Directory ---------- - -The server API for directory queries is also based on Federation Queries. - -{{directory_ss_http_api}} +{{query_ss_http_api}} Send-to-device messaging ------------------------ From d914c402e24e69536902703a305dbf4391c57303 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 16:03:35 -0600 Subject: [PATCH 15/52] Improve the profile query to have more sane responses This is a mix of Synapse and Dendrite behaviour, mostly Dendrite. Synapse returns `null` for field values that aren't set, however Dendrite just doesn't return them and instead opts for an empty object. Further, synapse is lacking in error codes in this area. Dendrite does some additional validation on this API which introduces more errors for bad requests, instead of defaulting to empty objects/200 OK responses. Likewise, Dendrite returns a 404 when the user is not found while Synapse returns 200 OK/empty object. --- api/server-server/query.yaml | 38 ++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/api/server-server/query.yaml b/api/server-server/query.yaml index 23b9a575..4599d130 100644 --- a/api/server-server/query.yaml +++ b/api/server-server/query.yaml @@ -131,7 +131,8 @@ paths: x-example: "@someone:example.org" - in: query name: field - type: string + type: enum + enum: ['displayname', 'avatar_url'] description: |- The field to query. If specified, the server will only return the given field in the response. If not specified, the server will return the full profile for @@ -144,26 +145,22 @@ paths: the response should include the fields of the user's profile that can be made public, such as the display name and avatar. - If the ``field`` is for a field that the server does not recognize, an empty object - should be returned. If the ``field`` is recognized, but the user does not have it - set on their profile, ``null`` should be returned for the value of that field. - - If the user does not exist, an empty object should be returned regardless of the - ``field`` being queried. + If the user does not have a particular field set on their profile, the server + should exclude it from the response body or give it the value ``null``. schema: type: object properties: displayname: type: string description: |- - The display name of the user. If the user does not have a display name set, - this should be the value ``null``. + The display name of the user. May be omitted if the user does not have a + display name set. x-example: "John Doe" avatar_url: type: string description: |- - The avatar URL for the user's avatar. If the user does not have an avatar - set, this should be the value ``null``. + The avatar URL for the user's avatar. May be omitted if the user does not + have an avatar set. x-example: "mxc://matrix.org/MyC00lAvatar" examples: application/json: { @@ -171,11 +168,26 @@ paths: "avatar_url": "mxc://matrix.org/MyC00lAvatar" } 400: - description: The user does not belong to the server. + description: |- + The request was missing parameters or had invalid values for the parameters. This + can happen for: + + * The user not being hosted on the homeserver, + * Lack of a ``user_id`` in the request, or + * The ``field`` requested not being an allowed value. schema: $ref: "../client-server/definitions/errors/error.yaml" examples: application/json: { - "errcode": "M_UNKNOWN", + "errcode": "M_INVALID_ARGUMENT_VALUE", "error": "User is not hosted on this homeserver." } + 404: + description: The user does not exist or does not have a profile. + schema: + $ref: "../client-server/definitions/errors/error.yaml" + examples: + application/json: { + "errcode": "M_NOT_FOUND", + "error": "User does not exist." + } From 8e97b0ca8174190f8bca6170cd39f92d09598d87 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 18 Jul 2018 10:51:28 -0600 Subject: [PATCH 16/52] Improve the server key exchange portion of the s2s specification Most of the text has been shuffled into the swagger definitions to bring it closer to where it matters. This also attempts to clarify what is out in the wild. Most importantly, the first version of the key exchange is outright removed from the specification. Other research points/questions are: * What is a "Key ID"? * https://github.com/matrix-org/synapse/blob/1241156c82644d5609f45659607a356af5d8fe08/synapse/rest/key/v2/local_key_resource.py#L81-L83 * https://github.com/matrix-org/synapse/blob/1241156c82644d5609f45659607a356af5d8fe08/synapse/rest/key/v2/local_key_resource.py#L88-L91 * Returning a cached response if the server throws a 400, 500, or otherwise not-offline status code * https://github.com/matrix-org/synapse/blob/1241156c82644d5609f45659607a356af5d8fe08/synapse/rest/key/v2/remote_key_resource.py#L227-L229 * `minimum_valid_until_ts` default * This branch of the ladder: https://github.com/matrix-org/synapse/blob/1241156c82644d5609f45659607a356af5d8fe08/synapse/rest/key/v2/remote_key_resource.py#L192 * Returning empty arrays when querying offline/no servers * Queried by hand against matrix.org as a notary server with a fake domain name to query * Returning all keys even when querying for specific keys * Queried by hand using matrix.org as a notary server against a server publishing multiple keys. The examples and descriptions were also improved as part of this commit. --- api/server-server/definitions/keys.yaml | 46 ++++--- .../definitions/keys_query_response.yaml | 4 +- api/server-server/examples/server_key.json | 12 +- .../examples/server_key_notary_signed.json | 11 ++ api/server-server/keys_query.yaml | 69 +++++++--- api/server-server/keys_server.yaml | 30 ++++- specification/server_server_api.rst | 120 +++--------------- 7 files changed, 143 insertions(+), 149 deletions(-) create mode 100644 api/server-server/examples/server_key_notary_signed.json diff --git a/api/server-server/definitions/keys.yaml b/api/server-server/definitions/keys.yaml index b08f2465..011ad6fd 100644 --- a/api/server-server/definitions/keys.yaml +++ b/api/server-server/definitions/keys.yaml @@ -20,50 +20,60 @@ properties: server_name: type: string description: DNS name of the homeserver. - required: true # TODO: Verify + required: true example: "example.org" verify_keys: type: object - description: Public keys of the homeserver for verifying digital signatures. - required: true # TODO: Verify + description: |- + Public keys of the homeserver for verifying digital signatures. + + The object's key is the algorithm and version combined (``ed25519`` being the + algorithm and ``abc123`` being the version in the example below). Together, + this forms the Key ID. + required: true additionalProperties: type: object title: Verify Key example: { - "ed25519:auto2": { - "key": "Base+64+Encoded+Signature+Verification+Key" + "ed25519:abc123": { + "key": "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA==" } } properties: key: type: string - description: The key + description: The `Unpadded Base64`_ encoded key. required: true - example: "Base+64+Encoded+Signature+Verification+Key" + example: "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA==" old_verify_keys: type: object - description: The public keys that the server used to use and when it stopped using them. + description: |- + The public keys that the server used to use and when it stopped using them. + + The object's key is the algorithm and version combined (``ed25519`` being the + algorithm and ``0ldK3y`` being the version in the example below). Together, + this forms the Key ID. additionalProperties: type: object title: Old Verify Key example: { - "ed25519:auto1": { + "ed25519:0ldK3y": { "expired_ts": 922834800000, - "key": "Base+64+Encoded+Signature+Verification+Key" + "key": "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg==" } } properties: expired_ts: type: integer format: int64 - description: The expiration time. + description: POSIX timestamp for when this key expired. required: true example: 922834800000 key: type: string - description: The key. + description: The `Unpadded Base64`_ encoded key. required: true - example: "Base+64+Encoded+Signature+Verification+Key" + example: "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg==" signatures: type: object description: Digital signatures for this object signed using the ``verify_keys``. @@ -72,7 +82,7 @@ properties: title: Signed Server example: { "example.org": { - "ad25519:auto2": "Base+64+Encoded+Signature+Verification+Key" + "ad25519:abc123": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU=" } } additionalProperties: @@ -87,10 +97,12 @@ properties: properties: sha256: type: string - description: The encoded fingerprint. - example: Base+64+Encoded+SHA-256-Fingerprint + description: The `Unpadded Base64`_ encoded fingerprint + example: "VGhpcyBpcyBoYXNoIHdoaWNoIHNob3VsZCBiZSBieXRlcw==" valid_until_ts: type: integer format: int64 - description: POSIX timestamp when the list of valid keys should be refreshed. + description: |- + POSIX timestamp when the list of valid keys should be refreshed. Keys used beyond this + timestamp are no longer valid. example: 1052262000000 diff --git a/api/server-server/definitions/keys_query_response.yaml b/api/server-server/definitions/keys_query_response.yaml index bf2d238f..52ad506c 100644 --- a/api/server-server/definitions/keys_query_response.yaml +++ b/api/server-server/definitions/keys_query_response.yaml @@ -15,13 +15,13 @@ type: object description: Server keys example: { "server_keys": [{ - $ref: "../examples/server_key.json" + $ref: "../examples/server_key_notary_signed.json" }] } properties: server_keys: type: array title: Server Keys - description: The server keys. + description: The queried server's keys, signed by the notary server. items: $ref: "keys.yaml" diff --git a/api/server-server/examples/server_key.json b/api/server-server/examples/server_key.json index a3934bd3..8a82d76e 100644 --- a/api/server-server/examples/server_key.json +++ b/api/server-server/examples/server_key.json @@ -1,23 +1,23 @@ { "server_name": "example.org", "verify_keys": { - "ed25519:auto2": { - "key": "Base+64+Encoded+Signature+Verification+Key" + "ed25519:abc123": { + "key": "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA==" } }, "old_verify_keys": { - "ed25519:auto1": { + "ed25519:0ldk3y": { "expired_ts": 922834800000, - "key": "Base+64+Encoded+Old+Verify+Key" + "key": "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg==" } }, "signatures": { "example.org": { - "ed25519:auto2": "Base+64+Encoded+Signature" + "ed25519:auto2": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU=" } }, "tls_fingerprints": [{ - "sha256": "Base+64+Encoded+SHA-256-Fingerprint" + "sha256": "VGhpcyBpcyBoYXNoIHdoaWNoIHNob3VsZCBiZSBieXRlcw==" }], "valid_until_ts": 1052262000000 } \ No newline at end of file diff --git a/api/server-server/examples/server_key_notary_signed.json b/api/server-server/examples/server_key_notary_signed.json new file mode 100644 index 00000000..f83bcc52 --- /dev/null +++ b/api/server-server/examples/server_key_notary_signed.json @@ -0,0 +1,11 @@ +{ + "$ref": "server_key.json", + "signatures": { + "example.org": { + "ed25519:abc123": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU=" + }, + "notary.server.com": { + "ed25519:010203": "VGhpcyBpcyBhbm90aGVyIHNpZ25hdHVyZQ==" + } + } +} \ No newline at end of file diff --git a/api/server-server/keys_query.yaml b/api/server-server/keys_query.yaml index f41cb35b..a1ea80fb 100644 --- a/api/server-server/keys_query.yaml +++ b/api/server-server/keys_query.yaml @@ -25,49 +25,60 @@ produces: paths: "/query/{serverName}/{keyId}": get: - summary: Retrieve a server key. - description: Retrieve a server key. - operationId: perspectivesKeyQuery + summary: Query for another server's keys + description: |- + Query for another server's keys. The receiving (notary) server must + sign the keys returned by the queried server. + operationId: getQueryKeys parameters: - in: path name: serverName type: string - description: Server name. + description: The server's DNS name to query required: true x-example: matrix.org - in: path name: keyId type: string - description: Key ID. - required: true - x-example: TODO # No examples in spec so far + description: |- + The key ID to look up. If omitted or empty, all the server's keys + are to be queried for. + required: false + x-example: "ed25519:abc123" - in: query name: minimum_valid_until_ts type: integer format: int64 - description: Minimum Valid Until Milliseconds. - required: true # TODO: Verify + description: |- + A millisecond POSIX timestamp indicating when the returned certificates + will need to be valid until to be useful to the requesting server. + + If not supplied, the current time as determined by the notary server is used. + required: false x-example: 1234567890 responses: 200: - description: The keys for the server + description: |- + The keys for the server, or an empty array if the server could not be reached + and no cached keys were available. schema: $ref: "definitions/keys_query_response.yaml" "/query": post: - summary: Retrieve a server key - description: Retrieve a server key. - operationId: bulkPerspectivesKeyQuery + summary: Query for several server's keys + description: |- + Query for keys from multiple servers in a batch format. The receiving (notary) + server must sign the keys returned by the queried servers. + operationId: postQueryKeys parameters: - in: body name: body schema: type: object - # TODO: Improve example example: { "server_keys": { - "{server_name}": { - "{key_id}": { + "example.org": { + "ed25519:abc123": { "minimum_valid_until_ts": 1234567890 } } @@ -76,7 +87,16 @@ paths: properties: server_keys: type: object - description: The query criteria. + description: |- + The query criteria. The outer ``string`` key on the object is the + server name (eg: ``matrix.org``). The inner ``string`` key is the + Key ID to query for the particular server. If no key IDs are given + to be queried, the notary server should query for all keys. If no + servers are given, the notary server must return an empty ``server_keys`` + array in the response. + + The notary server may return multiple keys regardless of the Key IDs + given. additionalProperties: type: object name: ServerName @@ -84,16 +104,25 @@ paths: additionalProperties: type: object title: Query Criteria - description: The server keys to query. + description: The server key IDs to query. properties: minimum_valid_until_ts: type: integer format: int64 - description: Minimum Valid Until MS. + description: |- + A millisecond POSIX timestamp indicating when the returned + certificates will need to be valid until to be useful to the + requesting server. + + If not supplied, the current time as determined by the notary + server is used. example: 1234567890 required: ['server_keys'] responses: 200: - description: The keys for the server. + description: |- + The keys for the queried servers, signed by the notary server. Servers which + are offline and have no cached keys will not be included in the result. This + may result in an empty array. schema: $ref: "definitions/keys_query_response.yaml" diff --git a/api/server-server/keys_server.yaml b/api/server-server/keys_server.yaml index 46beeebb..07a2c530 100644 --- a/api/server-server/keys_server.yaml +++ b/api/server-server/keys_server.yaml @@ -25,18 +25,38 @@ produces: paths: "/server/{keyId}": get: - summary: Get the server's key - description: Get the server's key. + summary: Get the homeserver's public key(s) + description: |- + Gets the homeserver's published TLS fingerprints and signing keys. + The homeserver may have any number of active keys and may have a + number of old keys. Homeservers SHOULD return a single JSON object + listing all of its keys, regardless of the ``keyId`` path argument. + This is to reduce the number of round trips needed to discover the + relevant keys for a homeserver. + + Intermediate notary servers should cache a response for half of its + lifetime to avoid serving a stale response. Originating servers should + avoid returning responses that expire in less than an hour to avoid + repeated reqests for a certificate that is about to expire. Requesting + servers should limit how frequently they query for certificates to + avoid flooding a server with requests. + + If the server fails to respond to this request, intermediate notary + servers should continue to return the last response they received + from the server so that the signatures of old events can still be + checked. operationId: getServerKey parameters: - in: path name: keyId type: string - description: Key ID + description: |- + The key ID to look up. If omitted or empty, all server keys are + to be returned. required: false - x-example: TODO # No examples in the spec so far + x-example: "ed25519:abc123" responses: 200: - description: The server's keys. + description: The homeserver's keys schema: $ref: "definitions/keys.yaml" diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f1825f27..4cf92e9b 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -106,15 +106,17 @@ Server implementation Retrieving Server Keys ~~~~~~~~~~~~~~~~~~~~~~ -Version 2 -+++++++++ +.. NOTE:: + There was once a "version 1" of the key exchange. It has been removed from the + specification due to lack of significance. It may be reviewed `here + `_. -Each homeserver publishes its public keys under ``/_matrix/key/v2/server/``. -Homeservers query for keys by either getting ``/_matrix/key/v2/server/`` +Each homeserver publishes its public keys under ``/_matrix/key/v2/server/{keyId}`. +Homeservers query for keys by either getting ``/_matrix/key/v2/server/{keyId}`` directly or by querying an intermediate notary server using a -``/_matrix/key/v2/query`` API. Intermediate notary servers query the -``/_matrix/key/v2/server/`` API on behalf of another server and sign the -response with their own key. A server may query multiple notary servers to +``/_matrix/key/v2/query/{serverName}/{keyId}`` API. Intermediate notary servers +query the ``/_matrix/key/v2/server/{keyId}`` API on behalf of another server and +sign the response with their own key. A server may query multiple notary servers to ensure that they all report the same public keys. This approach is borrowed from the `Perspectives Project`_, but modified to @@ -126,113 +128,33 @@ server by querying other servers. .. _Perspectives Project: https://web.archive.org/web/20170702024706/https://perspectives-project.org/ Publishing Keys -^^^^^^^^^^^^^^^ ++++++++++++++++ Homeservers publish the allowed TLS fingerprints and signing keys in a JSON object at ``/_matrix/key/v2/server/{key_id}``. The response contains a list of ``verify_keys`` that are valid for signing federation requests made by the -server and for signing events. It contains a list of ``old_verify_keys`` which +homeserver and for signing events. It contains a list of ``old_verify_keys`` which are only valid for signing events. Finally the response contains a list of TLS -certificate fingerprints to validate any connection made to the server. - -A server may have multiple keys active at a given time. A server may have any -number of old keys. It is recommended that servers return a single JSON -response listing all of its keys whenever any ``key_id`` is requested to reduce -the number of round trips needed to discover the relevant keys for a server. -However a server may return different responses for a different ``key_id``. - -The ``tls_certificates`` field contains a list of hashes of the X.509 TLS -certificates currently used by the server. The list must include SHA-256 hashes -for every certificate currently in use by the server. These fingerprints are -valid until the millisecond POSIX timestamp in ``valid_until_ts``. - -The ``verify_keys`` can be used to sign requests and events made by the server -until the millisecond POSIX timestamp in ``valid_until_ts``. If a homeserver -receives an event with a ``origin_server_ts`` after the ``valid_until_ts`` then -it should request that ``key_id`` for the originating server to check whether -the key has expired. - -The ``old_verify_keys`` can be used to sign events with an ``origin_server_ts`` -before the ``expired_ts``. The ``expired_ts`` is a millisecond POSIX timestamp -of when the originating server stopped using that key. - -Intermediate notary servers should cache a response for half of its remaining -lifetime to avoid serving a stale response. Originating servers should avoid -returning responses that expire in less than an hour to avoid repeated requests -for a certificate that is about to expire. Requesting servers should limit how -frequently they query for certificates to avoid flooding a server with -requests. - -If a server goes offline intermediate notary servers should continue to return -the last response they received from that server so that the signatures of old -events sent by that server can still be checked. +certificate fingerprints to validate any connection made to the homeserver. {{keys_server_ss_http_api}} Querying Keys Through Another Server -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +++++++++++++++++++++++++++++++++++++ -Servers may offer a query API ``/_matrix/key/v2/query/`` for getting the keys -for another server. This API can be used to GET a list of JSON objects for a -given server or to POST a bulk query for a number of keys from a number of -servers. Either way the response is a list of JSON objects containing the -JSON published by the server under ``/_matrix/key/v2/server/`` signed by -both the originating server and by this server. +Servers may query another server's keys through a notary server. The notary +server may be another homeserver. The notary server will retrieve keys from +the queried servers through use of the ``/_matrix/key/v2/server/{keyId}`` +API. The notary server will additionally sign the response from the queried +server before returning the results. -The ``minimum_valid_until_ts`` is a millisecond POSIX timestamp indicating -when the returned certificate will need to be valid until to be useful to the -requesting server. This can be set using the maximum ``origin_server_ts`` of -a batch of events that a requesting server is trying to validate. This allows -an intermediate notary server to give a prompt cached response even if the -originating server is offline. - -This API can return keys for servers that are offline by using cached responses -taken from when the server was online. Keys can be queried from multiple -servers to mitigate against DNS spoofing. +Notary servers can return keys for servers that are offline or having issues +serving their own keys by using cached responses. Keys can be queried from +multiple servers to mitigate against DNS spoofing. {{keys_query_ss_http_api}} -Version 1 -+++++++++ -.. WARNING:: - Version 1 of key distribution is obsolete. - - -Homeservers publish their TLS certificates and signing keys in a JSON object -at ``/_matrix/key/v1``. - -==================== =================== ====================================== - Key Type Description -==================== =================== ====================================== -``server_name`` String DNS name of the homeserver. -``verify_keys`` Object Public keys of the homeserver for - verifying digital signatures. -``signatures`` Object Digital signatures for this object - signed using the ``verify_keys``. -``tls_certificate`` String The X.509 TLS certificate used by this - this server encoded as `Unpadded Base64`_. -==================== =================== ====================================== - -.. code:: json - - { - "server_name": "example.org", - "signatures": { - "example.org": { - "ed25519:auto": "Base+64+Encoded+Signature" - } - }, - "tls_certificate": "Base+64+Encoded+DER+Encoded+X509+TLS+Certificate", - "verify_keys": { - "ed25519:auto": "Base+64+Encoded+Signature+Verification+Key" - } - } - -When fetching the keys for a server the client should check that the TLS -certificate in the JSON matches the TLS server certificate for the connection -and should check that the JSON signatures are correct for the supplied -``verify_keys``. Transactions ------------ From c2f1c6e78d4d92aa77e9df131dfc9234a5a3e614 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 18 Jul 2018 16:21:48 -0600 Subject: [PATCH 17/52] Improve the joining rooms handshake documentation There isn't a whole lot to this section that needed work. The section overall lost the table schema in favour of having the endpoints close by. The directory query is improved in https://github.com/matrix-org/matrix-doc/pull/1443 --- api/server-server/joins.yaml | 29 +++++--- specification/server_server_api.rst | 100 ++++++---------------------- 2 files changed, 39 insertions(+), 90 deletions(-) diff --git a/api/server-server/joins.yaml b/api/server-server/joins.yaml index eaf14e71..472d63bf 100644 --- a/api/server-server/joins.yaml +++ b/api/server-server/joins.yaml @@ -29,7 +29,6 @@ paths: description: |- Asks the receiving server to return information that the sending server will need to prepare a join event to get into the room. - This is part of the `Joining Rooms`_ handshake. operationId: makeJoin parameters: - in: path @@ -95,7 +94,9 @@ paths: type: array description: |- An event reference list containing the authorization events that would - allow the member to join the room. + allow the member to join the room. This should normally be the + ``m.room.create``, ``m.room.power_levels``, and ``m.room.join_rules`` + events. items: type: array maxItems: 2 @@ -128,7 +129,12 @@ paths: "state_key": "@someone:example.org", "content": { "membership": "join" - } + }, + "auth_events": [ + ["$room_cre4te_3vent:matrix.org", {"sha256": "abase64encodedsha256hashshouldbe43byteslong"}], + ["$room_j0in_rul3s_3vent:matrix.org", {"sha256": "abase64encodedsha256hashshouldbe43byteslong"}], + ["$room_p0wer_l3vels_3vent:matrix.org", {"sha256": "abase64encodedsha256hashshouldbe43byteslong"}] + ] } "/send_join/{roomId}/{eventId}": put: @@ -250,27 +256,30 @@ paths: title: Room State description: The state for the room. properties: + origin: + type: string + description: The resident server's DNS name. auth_chain: type: array description: The auth chain. items: type: object - properties: {} - # TODO: Verify schema + schema: + $ref: "definitions/pdu.yaml" state: type: array description: The room state. items: type: object - properties: {} - # TODO: Verify schema + schema: + $ref: "definitions/pdu.yaml" required: ["auth_chain", "state"] examples: application/json: [ 200, { - # TODO: Use the appropriate refs (see TODOs in schema) - "auth_chain": [], - "state": [] + "origin": "matrix.org", + "auth_chain": [{"$ref": "examples/pdu.json"}], + "state": [{"$ref": "examples/pdu.json"}] } ] diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f1825f27..d819b072 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -688,9 +688,6 @@ All these URLs are name-spaced within a prefix of:: {{query_general_ss_http_api}} - -{{joins_ss_http_api}} - Joining Rooms ------------- @@ -742,94 +739,34 @@ homeservers, though most in practice will use just two. <---------- join response The first part of the handshake usually involves using the directory server to -request the room ID and join candidates. This is covered in more detail on the -directory server documentation, below. In the case of a new user joining a -room as a result of a received invite, the joining user's homeserver could -optimise this step away by picking the origin server of that invite message as -the join candidate. However, the joining server should be aware that the origin -server of the invite might since have left the room, so should be prepared to -fall back on the regular join flow if this optimisation fails. +request the room ID and join candidates through the |/query/directory|_ +API endpoint. In the case of a new user joining a room as a result of a received +invite, the joining user's homeserver could optimise this step away by picking +the origin server of that invite message as the join candidate. However, the +joining server should be aware that the origin server of the invite might since +have left the room, so should be prepared to fall back on the regular join flow +if this optimisation fails. Once the joining server has the room ID and the join candidates, it then needs to obtain enough information about the room to fill in the required fields of the ``m.room.member`` event. It obtains this by selecting a resident from the -candidate list, and requesting the ``make_join`` endpoint using a ``GET`` -request, specifying the room ID and the user ID of the new member who is -attempting to join. +candidate list, and using the ``GET /make_join`` endpoint. The resident server +will then reply with enough information for the joining server to fill in the +event. -The resident server replies to this request with a JSON-encoded object having a -single key called ``event``; within this is an object whose fields contain some -of the information that the joining server will need. Despite its name, this -object is not a full event; notably it does not need to be hashed or signed by -the resident homeserver. The required fields are: - -======================== ============ ========================================= - Key Type Description -======================== ============ ========================================= -``type`` String The value ``m.room.member``. -``auth_events`` List An event-reference list containing the - authorization events that would allow - this member to join. -``content`` Object The event content. -``depth`` Integer (this field must be present but is - ignored; it may be 0) -``origin`` String The name of the resident homeserver. -``origin_server_ts`` Integer A timestamp added by the resident - homeserver. -``prev_events`` List An event-reference list containing the - immediate predecessor events. -``room_id`` String The room ID of the room. -``sender`` String The user ID of the joining member. -``state_key`` String The user ID of the joining member. -======================== ============ ========================================= - -The ``content`` field itself must be an object, containing: - -======================== ============ ========================================= - Key Type Description -======================== ============ ========================================= -``membership`` String The value ``join``. -======================== ============ ========================================= - -The joining server now has sufficient information to construct the real join -event from these protoevent fields. It copies the values of most of them, -adding (or replacing) the following fields: - -======================== ============ ========================================= - Key Type Description -======================== ============ ========================================= -``event_id`` String A new event ID specified by the joining - homeserver. -``origin`` String The name of the joining homeserver. -``origin_server_ts`` Integer A timestamp added by the joining - homeserver. -======================== ============ ========================================= - -This will be a true event, so the joining server should apply the event-signing -algorithm to it, resulting in the addition of the ``hashes`` and ``signatures`` -fields. +The joining server is expected to add or replace the ``origin``, ``origin_server_ts``, +and ``event_id`` on the templated event received by the resident server. This +event is then signed by the joining server. To complete the join handshake, the joining server must now submit this new -event to an resident homeserver, by using the ``send_join`` endpoint. This is -invoked using the room ID and the event ID of the new member event. +event to an resident homeserver, by using the ``PUT /send_join`` endpoint. The resident homeserver then accepts this event into the room's event graph, and responds to the joining server with the full set of state for the -newly-joined room. This is returned as a two-element list, whose first element -is the integer 200, and whose second element is an object which contains the -following keys: +newly-joined room. The resident server must also send the event to other servers +participating in the room. -======================== ============ ========================================= - Key Type Description -======================== ============ ========================================= -``auth_chain`` List A list of events giving all of the events - in the auth chains for the join event and - the events in ``state``. -``state`` List A complete list of the prevailing state - events at the instant just before - accepting the new ``m.room.member`` - event. -======================== ============ ========================================= +{{joins_ss_http_api}} .. TODO-spec - (paul) I don't really understand why the full auth_chain events are given @@ -1306,6 +1243,9 @@ that are too long. [[TODO(markjh) We might want to allow the server to omit the output of well known hash functions like SHA-256 when none of the keys have been redacted]] +.. |/query/directory| replace:: ``/query/directory`` +.. _/query/directory: #get-matrix-federation-v1-query-directory + .. _`Invitation storage`: ../identity_service/unstable.html#invitation-storage .. _`Identity Service API`: ../identity_service/unstable.html .. _`Client-Server API`: ../client_server/unstable.html#m-room-member From 4f24d2101f5d84f5c69ad9665b94a3569facc831 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 09:51:08 -0600 Subject: [PATCH 18/52] Only get issues that are proposals when finding proposals Otherwise we end up with not-proposals showing up in the list. --- scripts/proposals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index eec6d652..867d35d5 100755 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -28,7 +28,7 @@ def getpage(url, page): def getbylabel(label): pagecount = 1 json = list() - urlbase = 'https://api.github.com/repos/matrix-org/matrix-doc/issues?state=all&labels=' + label + '&page=' + urlbase = 'https://api.github.com/repos/matrix-org/matrix-doc/issues?state=all&labels=proposal,' + label + '&page=' print(urlbase) json.extend(getpage(urlbase, 1)) for page in range(2, int(pagecount) + 1): From 9fdd8a6f964c70689bd7451d746e96164ce8ab5f Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 14:59:43 -0600 Subject: [PATCH 19/52] Document how leaving rooms/rejecting invites over federation works Fixes https://github.com/matrix-org/matrix-doc/issues/1401 This is very similar to the joining rooms handshake, and much of it is a near copy/paste of the make_join and send_join API. The major difference is the send_leave API doesn't return anything. References: * Handling of make_leave: https://github.com/matrix-org/synapse/blob/d69decd5c78c72abef50b597a689e2bc55a39702/synapse/handlers/federation.py#L1285-L1310 * send_leave route: https://github.com/matrix-org/synapse/blob/d69decd5c78c72abef50b597a689e2bc55a39702/synapse/federation/transport/client.py#L267 * make_leave route: https://github.com/matrix-org/synapse/blob/d69decd5c78c72abef50b597a689e2bc55a39702/synapse/federation/transport/server.py#L396 * send_leave returning nothing: https://github.com/matrix-org/synapse/blob/d69decd5c78c72abef50b597a689e2bc55a39702/synapse/handlers/federation.py#L1346 --- api/server-server/leaving.yaml | 266 ++++++++++++++++++++++++++++ specification/server_server_api.rst | 20 +++ 2 files changed, 286 insertions(+) create mode 100644 api/server-server/leaving.yaml diff --git a/api/server-server/leaving.yaml b/api/server-server/leaving.yaml new file mode 100644 index 00000000..11b0e01d --- /dev/null +++ b/api/server-server/leaving.yaml @@ -0,0 +1,266 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Leave Room API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/make_leave/{roomId}/{userId}": + get: + summary: Get information required to make a join event for a room + description: |- + Asks the receiving server to return information that the sending + server will need to prepare a leave event to get out of the room. + operationId: makeLeave + parameters: + - in: path + name: roomId + type: string + description: The room ID that is about to be left. + required: true + x-example: "!abc123:matrix.org" + - in: path + name: userId + type: string + description: The user ID the leave event will be for. + required: true + x-example: "@someone:example.org" + responses: + 200: + description: |- + An unsigned event that the sending server may use as a template + for when it calls ``/send_leave``. + schema: + allOf: + - $ref: "definitions/unsigned_pdu.yaml" + - type: object + properties: + # Note: we override a bunch of parameters to change their descriptions + sender: + type: string + description: The user ID of the leaving member. + example: "@someone:example.org" + origin: + type: string + description: The name of the resident homeserver. + example: "matrix.org" + origin_server_ts: + type: integer + format: int64 + description: A timestamp added by the resident homeserver. + example: 1234567890 + type: + type: string + description: The value ``m.room.member``. + example: "m.room.member" + state_key: + type: string + description: The user ID of the leaving member. + example: "@someone:example.org" + content: + type: object + title: Membership Event Content + description: The content of the event. + example: {"membership": "leave"} + properties: + membership: + type: string + description: The value ``leave``. + example: "leave" + required: ['membership'] + auth_events: + type: array + description: |- + An event reference list containing the authorization events that would + allow the member to leave the room. This should normally be the + ``m.room.create``, ``m.room.power_levels``, and ``m.room.join_rules`` + events. + items: + type: array + maxItems: 2 + minItems: 2 + items: + - type: string + title: Event ID + example: "$abc123:matrix.org" + - type: object + title: Event Hash + example: { + "sha256": "abase64encodedsha256hashshouldbe43byteslong" + } + properties: + sha256: + type: string + description: The event hash. + example: abase64encodedsha256hashshouldbe43byteslong + required: ['sha256'] + redacts: + type: string + description: Not used. + required: + # Every other field is already flagged as required by the $ref + - state_key + examples: + application/json: { + "$ref": "examples/unsigned_pdu.json", + "type": "m.room.member", + "state_key": "@someone:example.org", + "content": { + "membership": "leave" + }, + "auth_events": [ + ["$room_cre4te_3vent:matrix.org", {"sha256": "abase64encodedsha256hashshouldbe43byteslong"}], + ["$room_j0in_rul3s_3vent:matrix.org", {"sha256": "abase64encodedsha256hashshouldbe43byteslong"}], + ["$room_p0wer_l3vels_3vent:matrix.org", {"sha256": "abase64encodedsha256hashshouldbe43byteslong"}] + ] + } + 403: + description: |- + The request is not authorized. This could mean that the user is not in the room. + schema: + $ref: "../client-server/definitions/errors/error.yaml" + examples: + application/json: { + "errcode": "M_FORBIDDEN", + "error": "User is not in the room." + } + "/send_leave/{roomId}/{eventId}": + put: + summary: Submit a signed leave event to a resident server + description: |- + Submits a signed leave event to the resident server for it + to accept it into the room's graph. + operationId: sendLeave + parameters: + - in: path + name: roomId + type: string + description: The room ID that is about to be left. + required: true + x-example: "!abc123:matrix.org" + - in: path + name: eventId + type: string + description: The event ID for the leave event. + required: true + x-example: "$abc123:example.org" + - in: body + name: body + type: object + required: true + schema: + allOf: + - $ref: "definitions/pdu.yaml" + - type: object + properties: + # Note: we override a bunch of parameters to change their descriptions + sender: + type: string + description: The user ID of the leaving member. + example: "@someone:example.org" + origin: + type: string + description: The name of the leaving homeserver. + example: "matrix.org" + origin_server_ts: + type: integer + format: int64 + description: A timestamp added by the leaving homeserver. + example: 1234567890 + type: + type: string + description: The value ``m.room.member``. + example: "m.room.member" + state_key: + type: string + description: The user ID of the leaving member. + example: "@someone:example.org" + content: + type: object + title: Membership Event Content + description: The content of the event. + example: {"membership": "leave"} + properties: + membership: + type: string + description: The value ``leave``. + example: "leave" + required: ['membership'] + depth: + type: integer + description: This field must be present but is ignored; it may be 0. + example: 12 + auth_events: + type: array + description: |- + An event reference list containing the authorization events that would + allow the member to leave the room. + items: + type: array + maxItems: 2 + minItems: 2 + items: + - type: string + title: Event ID + example: "$abc123:matrix.org" + - type: object + title: Event Hash + example: { + "sha256": "abase64encodedsha256hashshouldbe43byteslong" + } + properties: + sha256: + type: string + description: The event hash. + example: abase64encodedsha256hashshouldbe43byteslong + required: ['sha256'] + redacts: + type: string + description: Not used. + required: + # Every other field is already flagged as required by the $ref + - state_key + example: { + "$ref": "examples/pdu.json", + "type": "m.room.member", + "state_key": "@someone:example.org", + "content": { + "membership": "leave" + } + } + responses: + 200: + description: |- + An empty response to indicate the event was accepted into the graph by + the receiving homeserver. + schema: + type: array + minItems: 2 + maxItems: 2 + items: + - type: integer + description: The value ``200``. + example: 200 + - type: object + title: Empty Object + description: An empty object. + examples: + application/json: [200, {}] diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f1825f27..32b6486f 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -881,6 +881,26 @@ Inviting to a room {{invites_ss_http_api}} +Leaving Rooms (Rejecting Invites) +--------------------------------- + +Normally homeservers can send appropriate ``m.room.member`` events to have users +leave the room, or to reject local invites. Remote invites from other homeservers +do not involve the server in the graph and therefore need another approach to +reject the invite. Joining the room and promptly leaving is not recommended as +clients and servers will interpret that as accepting the invite, then leaving the +room rather than rejecting the invite. + +Similar to the `Joining Rooms`_ handshake, the server which wishes to leave the +room starts with sending a ``/make_leave`` request to a resident server. In the +case of rejecting invites, the resident server may be the server which sent the +invite. After receiving a template event from ``/make_leave``, the leaving server +signs the event and replaces the ``event_id`` with it's own. This is then sent to +the resident server via ``/send_leave``. The resident server will then send the +event to other servers in the room. + +{{leaving_ss_http_api}} + Third-party invites ------------------- From 2f71703427f87ea2d612025b50b37709202b639e Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 16:42:21 -0600 Subject: [PATCH 20/52] Deprecate key ID querying --- api/server-server/keys_query.yaml | 5 +++-- api/server-server/keys_server.yaml | 11 +++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/server-server/keys_query.yaml b/api/server-server/keys_query.yaml index a1ea80fb..c2800bc6 100644 --- a/api/server-server/keys_query.yaml +++ b/api/server-server/keys_query.yaml @@ -41,8 +41,9 @@ paths: name: keyId type: string description: |- - The key ID to look up. If omitted or empty, all the server's keys - are to be queried for. + **Deprecated**. Servers should not use this parameter and instead + opt to return all keys, not just the requested one. The key ID to + look up. required: false x-example: "ed25519:abc123" - in: query diff --git a/api/server-server/keys_server.yaml b/api/server-server/keys_server.yaml index 07a2c530..8734f2ed 100644 --- a/api/server-server/keys_server.yaml +++ b/api/server-server/keys_server.yaml @@ -29,10 +29,7 @@ paths: description: |- Gets the homeserver's published TLS fingerprints and signing keys. The homeserver may have any number of active keys and may have a - number of old keys. Homeservers SHOULD return a single JSON object - listing all of its keys, regardless of the ``keyId`` path argument. - This is to reduce the number of round trips needed to discover the - relevant keys for a homeserver. + number of old keys. Intermediate notary servers should cache a response for half of its lifetime to avoid serving a stale response. Originating servers should @@ -51,10 +48,12 @@ paths: name: keyId type: string description: |- - The key ID to look up. If omitted or empty, all server keys are - to be returned. + **Deprecated**. Servers should not use this parameter and instead + opt to return all keys, not just the requested one. The key ID to + look up. required: false x-example: "ed25519:abc123" + deprecated: true responses: 200: description: The homeserver's keys From 9dbb627d7b91ec795bf3d2ce5e555af65b6c180a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 16:42:40 -0600 Subject: [PATCH 21/52] Undo accidental changes to the operation ID --- api/server-server/keys_query.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/server-server/keys_query.yaml b/api/server-server/keys_query.yaml index c2800bc6..f6e2cb19 100644 --- a/api/server-server/keys_query.yaml +++ b/api/server-server/keys_query.yaml @@ -29,7 +29,7 @@ paths: description: |- Query for another server's keys. The receiving (notary) server must sign the keys returned by the queried server. - operationId: getQueryKeys + operationId: perspectivesKeyQuery parameters: - in: path name: serverName @@ -70,7 +70,7 @@ paths: description: |- Query for keys from multiple servers in a batch format. The receiving (notary) server must sign the keys returned by the queried servers. - operationId: postQueryKeys + operationId: bulkPerspectivesKeyQuery parameters: - in: body name: body From cad1db2a14aa7592b37a8225fe5bd307691d9700 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 16:43:16 -0600 Subject: [PATCH 22/52] Unpadded means unpadded --- api/server-server/definitions/keys.yaml | 16 ++++++++-------- .../examples/server_key_notary_signed.json | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/api/server-server/definitions/keys.yaml b/api/server-server/definitions/keys.yaml index 011ad6fd..2f154236 100644 --- a/api/server-server/definitions/keys.yaml +++ b/api/server-server/definitions/keys.yaml @@ -36,7 +36,7 @@ properties: title: Verify Key example: { "ed25519:abc123": { - "key": "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA==" + "key": "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA" } } properties: @@ -44,7 +44,7 @@ properties: type: string description: The `Unpadded Base64`_ encoded key. required: true - example: "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA==" + example: "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA" old_verify_keys: type: object description: |- @@ -59,7 +59,7 @@ properties: example: { "ed25519:0ldK3y": { "expired_ts": 922834800000, - "key": "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg==" + "key": "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg" } } properties: @@ -73,7 +73,7 @@ properties: type: string description: The `Unpadded Base64`_ encoded key. required: true - example: "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg==" + example: "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg" signatures: type: object description: Digital signatures for this object signed using the ``verify_keys``. @@ -82,7 +82,7 @@ properties: title: Signed Server example: { "example.org": { - "ad25519:abc123": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU=" + "ad25519:abc123": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU" } } additionalProperties: @@ -90,15 +90,15 @@ properties: name: Encoded Signature Verification Key tls_fingerprints: type: array - description: Hashes of X.509 TLS certificates used by this server encoded as `Unpadded Base64`_. + description: Hashes of X.509 TLS certificates used by this server. items: type: object title: TLS Fingerprint properties: sha256: type: string - description: The `Unpadded Base64`_ encoded fingerprint - example: "VGhpcyBpcyBoYXNoIHdoaWNoIHNob3VsZCBiZSBieXRlcw==" + description: The `Unpadded Base64`_ encoded fingerprint. + example: "VGhpcyBpcyBoYXNoIHdoaWNoIHNob3VsZCBiZSBieXRlcw" valid_until_ts: type: integer format: int64 diff --git a/api/server-server/examples/server_key_notary_signed.json b/api/server-server/examples/server_key_notary_signed.json index f83bcc52..d3a461ba 100644 --- a/api/server-server/examples/server_key_notary_signed.json +++ b/api/server-server/examples/server_key_notary_signed.json @@ -2,10 +2,10 @@ "$ref": "server_key.json", "signatures": { "example.org": { - "ed25519:abc123": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU=" + "ed25519:abc123": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU" }, "notary.server.com": { - "ed25519:010203": "VGhpcyBpcyBhbm90aGVyIHNpZ25hdHVyZQ==" + "ed25519:010203": "VGhpcyBpcyBhbm90aGVyIHNpZ25hdHVyZQ" } } } \ No newline at end of file From bdccfca72679a22bc67b5ad522ed7e324067f5e3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 16:43:43 -0600 Subject: [PATCH 23/52] Timestamps should be in milliseconds --- api/server-server/definitions/keys.yaml | 2 +- api/server-server/keys_query.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/server-server/definitions/keys.yaml b/api/server-server/definitions/keys.yaml index 2f154236..67908268 100644 --- a/api/server-server/definitions/keys.yaml +++ b/api/server-server/definitions/keys.yaml @@ -66,7 +66,7 @@ properties: expired_ts: type: integer format: int64 - description: POSIX timestamp for when this key expired. + description: POSIX timestamp in milliseconds for when this key expired. required: true example: 922834800000 key: diff --git a/api/server-server/keys_query.yaml b/api/server-server/keys_query.yaml index f6e2cb19..8fbe00dc 100644 --- a/api/server-server/keys_query.yaml +++ b/api/server-server/keys_query.yaml @@ -51,8 +51,8 @@ paths: type: integer format: int64 description: |- - A millisecond POSIX timestamp indicating when the returned certificates - will need to be valid until to be useful to the requesting server. + A millisecond POSIX timestamp in milliseconds indicating when the returned + certificates will need to be valid until to be useful to the requesting server. If not supplied, the current time as determined by the notary server is used. required: false @@ -111,9 +111,9 @@ paths: type: integer format: int64 description: |- - A millisecond POSIX timestamp indicating when the returned - certificates will need to be valid until to be useful to the - requesting server. + A millisecond POSIX timestamp in milliseconds indicating when + the returned certificates will need to be valid until to be + useful to the requesting server. If not supplied, the current time as determined by the notary server is used. From 8f1a4ae0ea8cfb5192911f862c55c029aac175de Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 16:43:51 -0600 Subject: [PATCH 24/52] Formatting --- specification/server_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 4cf92e9b..3bdc5b2b 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -111,7 +111,7 @@ Retrieving Server Keys specification due to lack of significance. It may be reviewed `here `_. -Each homeserver publishes its public keys under ``/_matrix/key/v2/server/{keyId}`. +Each homeserver publishes its public keys under ``/_matrix/key/v2/server/{keyId}``. Homeservers query for keys by either getting ``/_matrix/key/v2/server/{keyId}`` directly or by querying an intermediate notary server using a ``/_matrix/key/v2/query/{serverName}/{keyId}`` API. Intermediate notary servers From e27f4a69a05a893ca5e5aec6df4269d6f93fcdac Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 16:46:13 -0600 Subject: [PATCH 25/52] Key versions must be [0-9a-zA-Z_] --- api/server-server/definitions/keys.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/server-server/definitions/keys.yaml b/api/server-server/definitions/keys.yaml index 67908268..5fba4d18 100644 --- a/api/server-server/definitions/keys.yaml +++ b/api/server-server/definitions/keys.yaml @@ -29,7 +29,8 @@ properties: The object's key is the algorithm and version combined (``ed25519`` being the algorithm and ``abc123`` being the version in the example below). Together, - this forms the Key ID. + this forms the Key ID. The version must have characters matching the regular + expression ``[a-zA-Z0-9_]``. required: true additionalProperties: type: object @@ -52,7 +53,8 @@ properties: The object's key is the algorithm and version combined (``ed25519`` being the algorithm and ``0ldK3y`` being the version in the example below). Together, - this forms the Key ID. + this forms the Key ID. The version must have characters matching the regular + expression ``[a-zA-Z0-9_]``. additionalProperties: type: object title: Old Verify Key From cafd1a9ab3ebcac646a411f3fb0bc11725d61d04 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 16:46:33 -0600 Subject: [PATCH 26/52] Use more modern timestamps --- api/server-server/definitions/keys.yaml | 4 ++-- api/server-server/examples/server_key.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/server-server/definitions/keys.yaml b/api/server-server/definitions/keys.yaml index 5fba4d18..738e9e46 100644 --- a/api/server-server/definitions/keys.yaml +++ b/api/server-server/definitions/keys.yaml @@ -60,7 +60,7 @@ properties: title: Old Verify Key example: { "ed25519:0ldK3y": { - "expired_ts": 922834800000, + "expired_ts": 1532645052628, "key": "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg" } } @@ -70,7 +70,7 @@ properties: format: int64 description: POSIX timestamp in milliseconds for when this key expired. required: true - example: 922834800000 + example: 1532645052628 key: type: string description: The `Unpadded Base64`_ encoded key. diff --git a/api/server-server/examples/server_key.json b/api/server-server/examples/server_key.json index 8a82d76e..a7e4621b 100644 --- a/api/server-server/examples/server_key.json +++ b/api/server-server/examples/server_key.json @@ -7,7 +7,7 @@ }, "old_verify_keys": { "ed25519:0ldk3y": { - "expired_ts": 922834800000, + "expired_ts": 1532645052628, "key": "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg==" } }, @@ -19,5 +19,5 @@ "tls_fingerprints": [{ "sha256": "VGhpcyBpcyBoYXNoIHdoaWNoIHNob3VsZCBiZSBieXRlcw==" }], - "valid_until_ts": 1052262000000 + "valid_until_ts": 1652262000000 } \ No newline at end of file From e4fd088fcc7a9357c4551c544a0e84b4d0f3731e Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 27 Jul 2018 12:05:30 +0100 Subject: [PATCH 27/52] Fix room tags spec --- api/client-server/tags.yaml | 6 +++-- specification/modules/tags.rst | 48 ++++++++++++++++------------------ 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/api/client-server/tags.yaml b/api/client-server/tags.yaml index 9310b25f..42db5d9d 100644 --- a/api/client-server/tags.yaml +++ b/api/client-server/tags.yaml @@ -1,4 +1,5 @@ # Copyright 2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -64,8 +65,9 @@ paths: examples: application/json: { "tags": { - "work": {"order": "1"}, - "pinned": {} + "m.favourite": {} + "u.Work": {"order": "1"}, + "u.Customers": {} } } tags: diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 25e48ab3..c085391e 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -1,4 +1,5 @@ .. Copyright 2016 OpenMarket Ltd +.. Copyright 2018 New Vector Ltd .. .. Licensed under the Apache License, Version 2.0 (the "License"); .. you may not use this file except in compliance with the License. @@ -17,22 +18,19 @@ Room Tagging .. _module:tagging: -Users can add tags to rooms. Tags are short strings used to label rooms, e.g. -"work", "family". A room may have multiple tags. Tags are only visible to the -user that set them but are shared across all their devices. +Users can add tags to rooms. Tags are namespaced strings used to label rooms. +A room may have multiple tags. Tags are only visible to the user that set them +but are shared across all their devices. Events ------ The tags on a room are received as single ``m.tag`` event in the -``account_data`` section of a room in a ``/sync``. +``account_data`` section of a room. The content of the ``m.tag`` event is a +``tags`` key whose value is an object mapping the name of each tag to another +object. -The ``m.tag`` can also be received in a ``/events`` response or in the -``account_data`` section of a room in ``/initialSync``. ``m.tag`` -events appearing in ``/events`` will have a ``room_id`` with the room -the tags are for. - -Each tag has an associated JSON object with information about the tag, e.g how +The JSON object associated with each tag gives information about the tag, e.g how to order the rooms with a given tag. Ordering information is given under the ``order`` key as a number between 0 and @@ -43,25 +41,25 @@ after the rooms with that tag that have an ``order`` key. The name of a tag MUST not exceed 255 bytes. -The name of a tag should be human readable. When displaying tags for a room a -client should display this human readable name. When adding a tag for a room -a client may offer a list to choose from that includes all the tags that the -user has previously set on any of their rooms. - -Two special names are listed in the specification: - -* ``m.favourite`` -* ``m.lowpriority`` - -{{m_tag_event}} - Tags namespaces are defined in the following way, depending on how the client are expected to interpret them: -* The namespace ``m.*`` is reserved for tags defined in the current specification -* The namespace ``u.*`` is reserved for user-defined tags, and the client should not try to interpret as anything other than an utf8 string +* The namespace ``m.*`` is reserved for tags defined in the Matrix specification +* The namespace ``u.*`` is reserved for user-defined tags. The portion of the string after the ``u.`` + is defined to be the display name of this tag. No other semantics should be inferred from tags in + this namespace. * A client or app willing to use special tags for advanced functionnality should namespace them similarly to state keys: ``tld.name.*`` * Any tag in the ``tld.name.*`` form but not matching the namespace of the current client should be ignored -* Any tag not matching the previous rules should be interpreted as an user tag from the ``u.*`` namespace +* Any tag not matching the above rules should be interpreted as a user tag from the ``u.*`` namespace, as if + the name had already had ``u.`` stripped from the start (ie. the name of the tag is used as the + display name directly). + +Two special names are listed in the specification: +The following tags are defined in the ``m.*`` namespace: + +* ``m.favourite``: The user's favourite rooms. These should be shown with higher precedence than other rooms. +* ``m.lowpriority``: These should be shown with lower precedence than others. + +{{m_tag_event}} Client Behaviour ---------------- From f90ed4b77d19363667e11f8dd4555a49e3fe3700 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 27 Jul 2018 12:08:04 +0100 Subject: [PATCH 28/52] Make non-namespaced tags verboten --- specification/modules/tags.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index c085391e..bf985ede 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -51,7 +51,8 @@ Tags namespaces are defined in the following way, depending on how the client ar * Any tag in the ``tld.name.*`` form but not matching the namespace of the current client should be ignored * Any tag not matching the above rules should be interpreted as a user tag from the ``u.*`` namespace, as if the name had already had ``u.`` stripped from the start (ie. the name of the tag is used as the - display name directly). + display name directly). These non-namespaced tags are supported for historical reasons. New tags should use + one of the defined namespaces above. Two special names are listed in the specification: The following tags are defined in the ``m.*`` namespace: From 5ddf5374db3dc80fcb093946ff5c6c5465b5e75f Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 27 Jul 2018 12:24:29 +0100 Subject: [PATCH 29/52] Missing comma --- api/client-server/tags.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/tags.yaml b/api/client-server/tags.yaml index 42db5d9d..b7bafab6 100644 --- a/api/client-server/tags.yaml +++ b/api/client-server/tags.yaml @@ -65,7 +65,7 @@ paths: examples: application/json: { "tags": { - "m.favourite": {} + "m.favourite": {}, "u.Work": {"order": "1"}, "u.Customers": {} } From 2dc51d416dddad4932a78c9b0e1128fe68b985de Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 27 Jul 2018 12:27:30 +0100 Subject: [PATCH 30/52] Ignore stuff in `m.` you don't understand --- specification/modules/tags.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index bf985ede..8c74c55f 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -43,7 +43,8 @@ The name of a tag MUST not exceed 255 bytes. Tags namespaces are defined in the following way, depending on how the client are expected to interpret them: -* The namespace ``m.*`` is reserved for tags defined in the Matrix specification +* The namespace ``m.*`` is reserved for tags defined in the Matrix specification. Clients must ignore + any tags in this namespace they don't understand. * The namespace ``u.*`` is reserved for user-defined tags. The portion of the string after the ``u.`` is defined to be the display name of this tag. No other semantics should be inferred from tags in this namespace. From 73b0a03b9bb72d965afa235f8766101398a2057a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 27 Jul 2018 08:39:15 -0600 Subject: [PATCH 31/52] Remove more padding --- api/server-server/examples/server_key.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/server-server/examples/server_key.json b/api/server-server/examples/server_key.json index a7e4621b..bebd8445 100644 --- a/api/server-server/examples/server_key.json +++ b/api/server-server/examples/server_key.json @@ -2,22 +2,22 @@ "server_name": "example.org", "verify_keys": { "ed25519:abc123": { - "key": "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA==" + "key": "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA" } }, "old_verify_keys": { "ed25519:0ldk3y": { "expired_ts": 1532645052628, - "key": "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg==" + "key": "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg" } }, "signatures": { "example.org": { - "ed25519:auto2": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU=" + "ed25519:auto2": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU" } }, "tls_fingerprints": [{ - "sha256": "VGhpcyBpcyBoYXNoIHdoaWNoIHNob3VsZCBiZSBieXRlcw==" + "sha256": "VGhpcyBpcyBoYXNoIHdoaWNoIHNob3VsZCBiZSBieXRlcw" }], "valid_until_ts": 1652262000000 } \ No newline at end of file From 31687608dc90c6a77f4e39531576d59edd505d00 Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Sun, 29 Jul 2018 12:16:10 +0900 Subject: [PATCH 32/52] Factor out common code --- api/check_examples.py | 57 ++++++++++++++++--------------------------- 1 file changed, 21 insertions(+), 36 deletions(-) diff --git a/api/check_examples.py b/api/check_examples.py index fb9430b6..81a9298e 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -43,22 +43,25 @@ except ImportError as e: raise +def check_schema(filepath, example, schema): + # Setting the 'id' tells jsonschema where the file is so that it + # can correctly resolve relative $ref references in the schema + schema['id'] = "file://" + os.path.abspath(filepath) + example = resolve_references(filepath, example) + resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file}) + jsonschema.validate(example, schema, resolver=resolver) + + def check_parameter(filepath, request, parameter): schema = parameter.get("schema") example = schema.get('example') - fileurl = "file://" + os.path.abspath(filepath) if example and schema: try: print ("Checking request schema for: %r %r" % ( filepath, request )) - # Setting the 'id' tells jsonschema where the file is so that it - # can correctly resolve relative $ref references in the schema - schema['id'] = fileurl - example = resolve_references(filepath, example) - resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file}) - jsonschema.validate(example, schema, resolver=resolver) + check_schema(filepath, example, schema) except Exception as e: raise ValueError("Error validating JSON schema for %r" % ( request @@ -68,18 +71,12 @@ def check_parameter(filepath, request, parameter): def check_response(filepath, request, code, response): example = response.get('examples', {}).get('application/json') schema = response.get('schema') - fileurl = "file://" + os.path.abspath(filepath) if example and schema: try: print ("Checking response schema for: %r %r %r" % ( filepath, request, code )) - # Setting the 'id' tells jsonschema where the file is so that it - # can correctly resolve relative $ref references in the schema - schema['id'] = fileurl - example = resolve_references(filepath, example) - resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file}) - jsonschema.validate(example, schema, resolver=resolver) + check_schema(filepath, example, schema) except Exception as e: raise ValueError("Error validating JSON schema for %r %r" % ( request, code @@ -127,30 +124,18 @@ def resolve_references(path, schema): return schema -def load_yaml(path): - if not path.startswith("file:///"): - raise Exception("Bad ref: %s" % (path,)) - path = path[len("file://"):] - with open(path, "r") as f: - return yaml.load(f) - - -def load_json(path): - if not path.startswith("file:///"): - raise Exception("Bad ref: %s" % (path,)) - path = path[len("file://"):] - with open(path, "r") as f: - return json.load(f) - - def load_file(path): print("Loading reference: %s" % path) - if path.endswith(".json"): - return load_json(path) - else: - # We have to assume it's YAML because some of the YAML examples - # do not have file extensions. - return load_yaml(path) + if not path.startswith("file:///"): + raise Exception("Bad ref: %s" % (path,)) + path = path[len("file://"):] + with open(path, "r") as f: + if path.endswith(".json"): + return json.load(f) + else: + # We have to assume it's YAML because some of the YAML examples + # do not have file extensions. + return yaml.load(f) if __name__ == '__main__': From 2c9f00d37aef000fc0a483d9d4bccac973124adb Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Sun, 29 Jul 2018 12:23:41 +0900 Subject: [PATCH 33/52] Follow PEP 8 --- api/check_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/check_examples.py b/api/check_examples.py index 81a9298e..3e791bae 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -58,7 +58,7 @@ def check_parameter(filepath, request, parameter): if example and schema: try: - print ("Checking request schema for: %r %r" % ( + print("Checking request schema for: %r %r" % ( filepath, request )) check_schema(filepath, example, schema) From d17ec7f1846a96844c750dac3935ccd32b323b74 Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Sun, 29 Jul 2018 12:25:11 +0900 Subject: [PATCH 34/52] Check that path starts with file://, not file:/// Because file:/// is not a thing on Windows. --- api/check_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/check_examples.py b/api/check_examples.py index 3e791bae..eb861fca 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -126,7 +126,7 @@ def resolve_references(path, schema): def load_file(path): print("Loading reference: %s" % path) - if not path.startswith("file:///"): + if not path.startswith("file://"): raise Exception("Bad ref: %s" % (path,)) path = path[len("file://"):] with open(path, "r") as f: From c6bfac41c535a9608a67181f43864bafd3ae6d16 Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Sun, 29 Jul 2018 12:30:25 +0900 Subject: [PATCH 35/52] Bypass jsonschema references resolver jsonschema's resolver doesn't work with local files on Windows. resolve_references only works with the local files (on Windows as well) but that's exactly what's needed for the case (as long as we don't have remote references, that is). Signed-off-by: Alexey Rusakov --- api/check_examples.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/api/check_examples.py b/api/check_examples.py index eb861fca..cb85cb50 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -44,16 +44,14 @@ except ImportError as e: def check_schema(filepath, example, schema): - # Setting the 'id' tells jsonschema where the file is so that it - # can correctly resolve relative $ref references in the schema - schema['id'] = "file://" + os.path.abspath(filepath) example = resolve_references(filepath, example) + schema = resolve_references(filepath, schema) resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file}) jsonschema.validate(example, schema, resolver=resolver) def check_parameter(filepath, request, parameter): - schema = parameter.get("schema") + schema = parameter.get("schema", {}) example = schema.get('example') if example and schema: From 346db48588612620f179e0b9560ac4d1d8a24729 Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Sun, 29 Jul 2018 14:04:07 +0900 Subject: [PATCH 36/52] Revert giving get() a default As per the PR review. Signed-off-by: Alexey Rusakov --- api/check_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/check_examples.py b/api/check_examples.py index cb85cb50..0fb275b1 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -51,7 +51,7 @@ def check_schema(filepath, example, schema): def check_parameter(filepath, request, parameter): - schema = parameter.get("schema", {}) + schema = parameter.get("schema") example = schema.get('example') if example and schema: From 0ff882fb3b3467daf74aca2fb55c4a4f12614272 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 13 Jul 2018 11:41:51 +0100 Subject: [PATCH 37/52] Field definitions for protocol metadata --- api/application-service/definitions/protocol_metadata.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/application-service/definitions/protocol_metadata.yaml b/api/application-service/definitions/protocol_metadata.yaml index 72264060..2b2c8f4e 100644 --- a/api/application-service/definitions/protocol_metadata.yaml +++ b/api/application-service/definitions/protocol_metadata.yaml @@ -13,6 +13,8 @@ # limitations under the License. type: object description: Dictionary of supported third party protocols. +additionalProperties: + $ref: protocol.yaml example: { "irc": { "user_fields": ["network", "nickname"], From 273acb277caa7020dbcea76e2707c45eca8670cd Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 13 Jul 2018 12:07:15 +0100 Subject: [PATCH 38/52] Change fields to use new extended query array syntax --- api/application-service/application_service.yaml | 4 ++-- api/client-server/third_party_lookup.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index b459e29f..138c75bd 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -270,7 +270,7 @@ paths: required: true x-example: irc - in: query - name: field1, field2... + name: fields... type: string description: |- One or more custom fields that are passed to the application @@ -321,7 +321,7 @@ paths: required: true x-example: irc - in: query - name: field1, field2... + name: field... type: string description: |- One or more custom fields that are passed to the application diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 55d4c70a..226eec21 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -122,7 +122,7 @@ paths: required: true x-example: irc - in: query - name: field1, field2... + name: fields... type: string description: |- One or more custom fields that are passed to the AS to help identify the user. From 3e4962f21152e31bce6a3ff44bb0c65b11c89616 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 13 Jul 2018 12:25:48 +0100 Subject: [PATCH 39/52] Document standardized extensions to OpenAPI v2 --- api/openapi_extensions.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 api/openapi_extensions.rst diff --git a/api/openapi_extensions.rst b/api/openapi_extensions.rst new file mode 100644 index 00000000..75c48687 --- /dev/null +++ b/api/openapi_extensions.rst @@ -0,0 +1,23 @@ +OpenAPI Extensions +================== + +For some functionality that is not directly provided by the OpenAPI v2 +specification, some extensions have been added that are to be consistent +across the specification. The defined extensions are listed below. Extensions +should not break parsers, however if extra functionality is required, aware +parsers should be able to take advantage of the added syntax. + +Extensible Query Parameters +--------------------------- + +If a unknown amount of query parameters can be added to a request, the +``name`` should be in form of ``something...``, with the trailing ellipses +representing the possibility of more fields. + +Example: + +.. code-block:: + + - in: query + name: fields... + type: string \ No newline at end of file From 53137a3c5318c1a8e50a988e7c511302cf923590 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 23 Jul 2018 14:38:52 +0100 Subject: [PATCH 40/52] c/p bug, fix operationIds, move rst docs to md --- .../application_service.yaml | 2 +- api/application-service/definitions/user.yaml | 2 +- api/client-server/third_party_lookup.yaml | 9 +++++---- ...pi_extensions.rst => openapi_extensions.md} | 18 ++++++++---------- 4 files changed, 15 insertions(+), 16 deletions(-) rename api/{openapi_extensions.rst => openapi_extensions.md} (65%) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index 138c75bd..6f2eed16 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -214,7 +214,7 @@ paths: This API is called by the homeserver when it wants to present clients with specific information about the various third party networks that an application service supports. - operationId: queryMetadata + operationId: getProtocolMetadata parameters: - in: path name: protocol diff --git a/api/application-service/definitions/user.yaml b/api/application-service/definitions/user.yaml index 5f8d0460..a7b2287e 100644 --- a/api/application-service/definitions/user.yaml +++ b/api/application-service/definitions/user.yaml @@ -27,5 +27,5 @@ properties: type: object example: "user": "jim" -title: Location +title: User type: object \ No newline at end of file diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 226eec21..ee14a137 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -34,7 +34,7 @@ paths: Fetches the overall metadata about protocols supported by the homeserver. Includes both the available protocols and all fields required for queries against each protocol. - operationId: queryMetadata + operationId: getProtocols responses: 200: description: The protocols supported by the homeserver. @@ -45,7 +45,7 @@ paths: summary: Retrieve metadata about a specific protocol that the homeserver supports. description: |- Fetches the metadata from the homeserver about a particular third party protocol. - operationId: queryMetadata + operationId: getProtocolMetadata parameters: - in: path name: protocol @@ -122,8 +122,9 @@ paths: required: true x-example: irc - in: query - name: fields... - type: string + name: fields + explode: true + type: object description: |- One or more custom fields that are passed to the AS to help identify the user. responses: diff --git a/api/openapi_extensions.rst b/api/openapi_extensions.md similarity index 65% rename from api/openapi_extensions.rst rename to api/openapi_extensions.md index 75c48687..4a01c477 100644 --- a/api/openapi_extensions.rst +++ b/api/openapi_extensions.md @@ -1,5 +1,4 @@ -OpenAPI Extensions -================== +# OpenAPI Extensions For some functionality that is not directly provided by the OpenAPI v2 specification, some extensions have been added that are to be consistent @@ -7,17 +6,16 @@ across the specification. The defined extensions are listed below. Extensions should not break parsers, however if extra functionality is required, aware parsers should be able to take advantage of the added syntax. -Extensible Query Parameters ---------------------------- +## Extensible Query Parameters -If a unknown amount of query parameters can be added to a request, the -``name`` should be in form of ``something...``, with the trailing ellipses -representing the possibility of more fields. +If a unknown amount of query parameters can be added to a request, the `name` +must be `fields...`, with the trailing ellipses representing the possibility +of more fields. Example: -.. code-block:: - +``` - in: query name: fields... - type: string \ No newline at end of file + type: string +``` \ No newline at end of file From d785ac78a355c85f68c82e3ff5e558378d2d1bc0 Mon Sep 17 00:00:00 2001 From: user Date: Mon, 30 Jul 2018 11:01:27 +0100 Subject: [PATCH 41/52] Remove explode and replace with TODO, require auth on CS --- api/client-server/third_party_lookup.yaml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index ee14a137..293a105b 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -35,6 +35,8 @@ paths: homeserver. Includes both the available protocols and all fields required for queries against each protocol. operationId: getProtocols + security: + - accessToken: [] responses: 200: description: The protocols supported by the homeserver. @@ -46,6 +48,8 @@ paths: description: |- Fetches the metadata from the homeserver about a particular third party protocol. operationId: getProtocolMetadata + security: + - accessToken: [] parameters: - in: path name: protocol @@ -80,6 +84,8 @@ paths: identifier. It should attempt to canonicalise the identifier as much as reasonably possible given the network type. operationId: queryLocationByProtocol + security: + - accessToken: [] parameters: - in: path name: protocol @@ -113,6 +119,8 @@ paths: Retrieve a Matrix User ID linked to a user on the third party service, given a set of user parameters. operationId: queryUserByProtocol + security: + - accessToken: [] parameters: - in: path name: protocol @@ -121,10 +129,10 @@ paths: The name of the protocol. required: true x-example: irc + # TODO: Change to 'explode' after OpenAPI/Swagger v3 update - in: query - name: fields - explode: true - type: object + name: field1, field2... + type: string description: |- One or more custom fields that are passed to the AS to help identify the user. responses: @@ -147,6 +155,8 @@ paths: Retreive an array of third party network locations from a Matrix room alias. operationId: queryLocationByAlias + security: + - accessToken: [] parameters: - in: query name: alias @@ -173,6 +183,8 @@ paths: description: |- Retreive an array of third party users from a Matrix User ID. operationId: queryUserByID + security: + - accessToken: [] parameters: - in: query name: userid @@ -192,4 +204,4 @@ paths: "errcode": "M_NOT_FOUND" } schema: - $ref: definitions/errors/error.yaml \ No newline at end of file + $ref: definitions/errors/error.yaml From a2e2ced03b1754ca71d69def19eb03e2deeb9766 Mon Sep 17 00:00:00 2001 From: user Date: Mon, 30 Jul 2018 11:09:13 +0100 Subject: [PATCH 42/52] Add reverse-lookup examples. --- api/client-server/third_party_lookup.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 293a105b..55b34a81 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -163,6 +163,7 @@ paths: type: string description: The Matrix room alias to look up. required: true + x-example: "#matrix:matrix.org" responses: 200: description: |- @@ -191,6 +192,7 @@ paths: type: string description: The Matrix User ID to look up. required: true + x-example: "@bob:matrix.org" responses: 200: description: |- From 5f8967c0742faa108f994771e2ecc0b0b42854da Mon Sep 17 00:00:00 2001 From: user Date: Mon, 30 Jul 2018 19:19:35 +0100 Subject: [PATCH 43/52] Move TODO. field1, field2... -> fields... --- api/application-service/application_service.yaml | 4 ++-- api/client-server/third_party_lookup.yaml | 3 +-- api/openapi_extensions.md | 4 +++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index 6f2eed16..42c0c0cf 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -321,7 +321,7 @@ paths: required: true x-example: irc - in: query - name: field... + name: fields... type: string description: |- One or more custom fields that are passed to the application @@ -446,4 +446,4 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - $ref: ../client-server/definitions/errors/error.yaml \ No newline at end of file + $ref: ../client-server/definitions/errors/error.yaml diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 55b34a81..cba9ce22 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -129,9 +129,8 @@ paths: The name of the protocol. required: true x-example: irc - # TODO: Change to 'explode' after OpenAPI/Swagger v3 update - in: query - name: field1, field2... + name: fields... type: string description: |- One or more custom fields that are passed to the AS to help identify the user. diff --git a/api/openapi_extensions.md b/api/openapi_extensions.md index 4a01c477..9f4745fd 100644 --- a/api/openapi_extensions.md +++ b/api/openapi_extensions.md @@ -8,6 +8,8 @@ parsers should be able to take advantage of the added syntax. ## Extensible Query Parameters + + If a unknown amount of query parameters can be added to a request, the `name` must be `fields...`, with the trailing ellipses representing the possibility of more fields. @@ -18,4 +20,4 @@ Example: - in: query name: fields... type: string -``` \ No newline at end of file +``` From 7679b4f1d168e4f5ab7c2e94d34923079782f234 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 30 Jul 2018 16:58:13 -0600 Subject: [PATCH 44/52] Improve documentation on how Transactions work The response is based upon various sections of the Synapse code in how it generates a response. There are no new fields added to the transaction. Originally, `previous_ids` and `pdu_failures` were to be documented however neither of these are used in the real world. --- .../definitions/transaction.yaml | 8 ++- api/server-server/transactions.yaml | 58 +++++++++++++++++-- specification/server_server_api.rst | 38 +----------- 3 files changed, 58 insertions(+), 46 deletions(-) diff --git a/api/server-server/definitions/transaction.yaml b/api/server-server/definitions/transaction.yaml index 930ddec1..7df8b646 100644 --- a/api/server-server/definitions/transaction.yaml +++ b/api/server-server/definitions/transaction.yaml @@ -25,11 +25,13 @@ properties: origin_server_ts: type: integer format: int64 - description: Timestamp in milliseconds on originating homeserver when this transaction started. - example: 1234567890 + description: |- + POSIX timestamp in milliseconds on originating homeserver when this + transaction started. + example: 1532991320875 pdus: type: array description: List of persistent updates to rooms. items: $ref: "pdu.yaml" -required: ['origin', 'origin_server_ts', 'pdus'] \ No newline at end of file +required: ['origin', 'origin_server_ts', 'pdus'] diff --git a/api/server-server/transactions.yaml b/api/server-server/transactions.yaml index 13ba6826..2a9180a1 100644 --- a/api/server-server/transactions.yaml +++ b/api/server-server/transactions.yaml @@ -30,16 +30,18 @@ paths: Push messages representing live activity to another server. The destination name will be set to that of the receiving server itself. Each embedded PDU in the transaction body will be processed. + + The sending server must wait and retry for a 200 OK response before sending a + transaction with a different ``txnId`` to the receiving server. operationId: sendTransaction parameters: - in: path name: txnId type: string - # TODO: "Overrides any ID given in the JSON body" - What does this mean? description: |- - The transaction ID. Overrides any ID given in the JSON body. + The transaction ID. required: true - x-example: TODO # No examples in the spec so far + x-example: S0meTransacti0nId - in: body name: body type: object @@ -51,7 +53,9 @@ paths: properties: edus: type: array - description: List of ephemeral messages. May be omitted if there are no ephemeral messages to be sent. + description: |- + List of ephemeral messages. May be omitted if there are no ephemeral + messages to be sent. items: $ref: "definitions/edu.yaml" example: { @@ -60,5 +64,47 @@ paths: } responses: 200: - # TODO: Spec this (and figure out what it is) - description: TODO + description: |- + The result of processing the transaction. The server is to use this response even in + the event of one or more PDUs failing to be processed. + schema: + type: array + minItems: 2 + maxItems: 2 + items: + - type: integer + description: The value ``200``. + example: 200 + - type: object + title: PDU Processing Results + description: The results for the processing of each PDU in the transaction. + properties: + pdus: + type: object + description: |- + The PDUs from the original transaction. The string key represents the ID of the + PDU (event) that was processed. + additionalProperties: + type: object + title: PDU Processing Result + description: Information about how the PDU was handled. + properties: + error: + type: string + description: |- + A human readable description about what went wrong in processing this PDU. + If no error is present, the PDU can be considered successfully handled. + example: "You are not allowed to send a message to this room." + required: ['pdus'] + examples: + application/json: [ + 200, + { + "pdus": { + "$successful_event:domain.com": {}, + "$failed_event:example.org": { + "error": "You are not allowed to send a message to this room." + } + } + } + ] diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f1825f27..a190d9d2 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -242,41 +242,7 @@ of Transaction messages, which are encoded as JSON objects, passed over an HTTP PUT request. A Transaction is meaningful only to the pair of homeservers that exchanged it; they are not globally-meaningful. -Each transaction has: - - An opaque transaction ID, unique among transactions from the same origin. - - A timestamp (UNIX epoch time in milliseconds) generated by its origin - server. - - An origin and destination server name. - - A list of PDUs and EDUs - the actual message payload that the Transaction - carries. - -Transaction Fields -~~~~~~~~~~~~~~~~~~ - -==================== =================== ====================================== - Key Type Description -==================== =================== ====================================== -``origin`` String **Required**. ``server_name`` of homeserver sending - this transaction. -``origin_server_ts`` Integer **Required**. Timestamp in milliseconds on - originating homeserver when this - transaction started. -``pdus`` List of Objects **Required**. List of persistent updates to rooms. -``edus`` List of Objects List of ephemeral messages. May be omitted - if there are no ephemeral messages to - be sent. -==================== =================== ====================================== - -Example: - -.. code:: json - - { - "origin_server_ts": 1404835423000, - "origin": "matrix.org", - "pdus": [...], - "edus": [...] - } +{{transactions_ss_http_api}} PDUs ---- @@ -682,8 +648,6 @@ All these URLs are name-spaced within a prefix of:: /_matrix/federation/v1/... -{{transactions_ss_http_api}} - {{events_ss_http_api}} {{query_general_ss_http_api}} From d7919b6989847297c0c29b00529d2b67bd9d64b0 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Tue, 31 Jul 2018 15:08:42 +0100 Subject: [PATCH 45/52] also scrape PRs when the full URL is pasted --- scripts/proposals.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index a8f58cae..d8c1aa24 100755 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -138,10 +138,15 @@ for label in labels: pr_list_formatted = set() pr_list = pr_list.group(1) for p in pr_list.split(","): - if not re.match(r"#\d", p.strip()): + if re.match(r"#\d", p.strip()): + prs.add(p.strip()) + pr_list_formatted.add("`PR" + str(p.strip()) + "`_") + elif re.match(r"https://github.com/matrix-org/matrix-doc/pulls/\d", p.strip()): + pr = "#" + p.strip().replace('https://github.com/matrix-org/matrix-doc/pulls/', '') + prs.add(pr) + pr_list_formatted.add("`PR" + str(pr) + "`_") + else: raise RuntimeWarning - prs.add(p.strip()) - pr_list_formatted.add("`PR" + str(p.strip()) + "`_") text_file.write(" - " + ', '.join(pr_list_formatted)) text_file.write("\n") else: From 9d474bb8195b3a61b1cad0330aa3f1adec7756e7 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 2 Aug 2018 12:44:54 -0600 Subject: [PATCH 46/52] Document event retrieval endpoints in more detail This also adds a previously-undocumented endpoint: /state_ids Backfill is technically not part of this section, however it is being left untouched to make the merge with #1469 easier (which moves it out of the file). Reference material: * Some calls to synapse on these endpoints with a relatively simple private room. --- api/server-server/events.yaml | 76 +++++++++++++++++++++++++++-- specification/server_server_api.rst | 12 ++++- 2 files changed, 83 insertions(+), 5 deletions(-) diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml index d540085d..f0c3250c 100644 --- a/api/server-server/events.yaml +++ b/api/server-server/events.yaml @@ -27,7 +27,7 @@ paths: get: summary: Get all the state of a given room description: |- - Retrieves a snapshot of the entire current state of the given room. + Retrieves a snapshot of a room's state at a given event. operationId: getRoomState parameters: - in: path @@ -36,11 +36,81 @@ paths: description: The room ID to get state for. required: true x-example: "!abc123:matrix.org" + - in: query + name: event_id + type: string + description: An event ID in the room to retrieve the state at. + required: true + x-example: "$helloworld:matrix.org" responses: 200: - description: The room state for the room (kept under ``pdus``). + description: |- + The fully resolved state for the room, including the authorization + chain for the events. schema: - $ref: "definitions/transaction.yaml" + type: object + properties: + auth_chain: + type: list + description: |- + The full set of authorization events that make up the state + of the room, and their authorization events, recursively. + items: + $ref: "definitions/pdu.yaml" + example: [{"$ref": "examples/pdu.json"}] + pdus: + type: list + description: |- + The fully resolved state of the room at the given event. + items: + $ref: "definitions/pdu.yaml" + example: [{"$ref": "examples/pdu.json"}] + required: ['auth_chain', 'pdus'] + "/state_ids/{roomId}": + get: + summary: Get all the state event IDs of a given room + description: |- + Retrieves a snapshot of a room's state at a given event, in the form of + event IDs. This performs the same function as calling ``/state/{roomId}``, + however this returns just the event IDs rather than the full events. + operationId: getRoomStateIds + parameters: + - in: path + name: roomId + type: string + description: The room ID to get state for. + required: true + x-example: "!abc123:matrix.org" + - in: query + name: event_id + type: string + description: An event ID in the room to retrieve the state at. + required: true + x-example: "$helloworld:matrix.org" + responses: + 200: + description: |- + The fully resolved state for the room, including the authorization + chain for the events. + schema: + type: object + properties: + auth_chain_ids: + type: list + description: |- + The full set of authorization events that make up the state + of the room, and their authorization events, recursively. + items: + type: string + example: ["$an_event:domain.com"] + pdu_ids: + type: list + description: |- + The fully resolved state of the room at the given event. + items: + type: string + example: ["$an_event:domain.com"] + required: ['auth_chain_ids', 'pdu_ids'] "/event/{eventId}": get: summary: Get a single event diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 3bdc5b2b..4a00db98 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -606,8 +606,6 @@ All these URLs are name-spaced within a prefix of:: {{transactions_ss_http_api}} -{{events_ss_http_api}} - {{query_general_ss_http_api}} @@ -798,6 +796,16 @@ that requested by the requester in the ``v`` parameter). Specify (or remark that it is unspecified) how the server handles divergent history. DFS? BFS? Anything weirder? +Retriving events +---------------- + +In some circumstances, a homeserver may be missing a particular event or information +about the room which cannot be easily determined from backfilling. These APIs provide +homeservers with the option of getting events and the state of the room at a given +point in the timeline. + +{{events_ss_http_api}} + Inviting to a room ------------------ From 8b7bc603670c2e81c73505b0987690b6b87fa269 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 2 Aug 2018 16:42:03 -0600 Subject: [PATCH 47/52] list -> array --- api/server-server/events.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml index f0c3250c..11c88e8d 100644 --- a/api/server-server/events.yaml +++ b/api/server-server/events.yaml @@ -51,7 +51,7 @@ paths: type: object properties: auth_chain: - type: list + type: array description: |- The full set of authorization events that make up the state of the room, and their authorization events, recursively. @@ -59,7 +59,7 @@ paths: $ref: "definitions/pdu.yaml" example: [{"$ref": "examples/pdu.json"}] pdus: - type: list + type: array description: |- The fully resolved state of the room at the given event. items: @@ -96,7 +96,7 @@ paths: type: object properties: auth_chain_ids: - type: list + type: array description: |- The full set of authorization events that make up the state of the room, and their authorization events, recursively. @@ -104,7 +104,7 @@ paths: type: string example: ["$an_event:domain.com"] pdu_ids: - type: list + type: array description: |- The fully resolved state of the room at the given event. items: From 5596243add762c0ee2ede0a5ca592f4437ee3ce2 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 2 Aug 2018 16:43:24 -0600 Subject: [PATCH 48/52] origin is required --- api/server-server/joins.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server-server/joins.yaml b/api/server-server/joins.yaml index 472d63bf..759361b7 100644 --- a/api/server-server/joins.yaml +++ b/api/server-server/joins.yaml @@ -273,7 +273,7 @@ paths: type: object schema: $ref: "definitions/pdu.yaml" - required: ["auth_chain", "state"] + required: ["auth_chain", "state", "origin"] examples: application/json: [ 200, From a9258ed1951d55e19f4ede02b7708e692a1bb1f0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 2 Aug 2018 16:43:29 -0600 Subject: [PATCH 49/52] an -> a --- specification/server_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index d819b072..8c3d85ea 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -759,7 +759,7 @@ and ``event_id`` on the templated event received by the resident server. This event is then signed by the joining server. To complete the join handshake, the joining server must now submit this new -event to an resident homeserver, by using the ``PUT /send_join`` endpoint. +event to a resident homeserver, by using the ``PUT /send_join`` endpoint. The resident homeserver then accepts this event into the room's event graph, and responds to the joining server with the full set of state for the From 05bb7e105015fc0a07ba7e0b2506f49502fd6ec4 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 2 Aug 2018 18:45:59 -0600 Subject: [PATCH 50/52] Spelling and word choice --- api/server-server/query.yaml | 6 +++--- specification/server_server_api.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/server-server/query.yaml b/api/server-server/query.yaml index 4599d130..2f4b8d1e 100644 --- a/api/server-server/query.yaml +++ b/api/server-server/query.yaml @@ -52,7 +52,7 @@ paths: that belong to the target server (idenfified by the DNS Name in the alias). The target server may not appear in the resident servers list. - Servers may wish to cache the response to this query to prevent requesting the + Servers may wish to cache the response to this query to avoid requesting the information too often. operationId: queryRoomDirectory parameters: @@ -76,7 +76,7 @@ paths: servers: type: array description: |- - An array of server names that are likely to hold then given room. This + An array of server names that are likely to hold the given room. This list may or may not include the server answering the query. items: type: string @@ -120,7 +120,7 @@ paths: for a given user. Homeservers should only query profiles for users that belong to the target server (identified by the DNS Name in the user ID). - Servers may wish to cache the response to this query to prevent requesting the + Servers may wish to cache the response to this query to avoid requesting the information too often. parameters: - in: query diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index bfc98fb8..36591356 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -968,7 +968,7 @@ Rejecting a presence invite:: Querying for information ------------------------ -Queries are a way to retrieve information from a homeserver abotu a resource, +Queries are a way to retrieve information from a homeserver about a resource, such as a user or room. The endpoints here are often called in conjunction with a request from a client on the client-server API in order to complete the call. From 0b313dbdd9d1a2f098db4240a98429b9ec01c686 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 2 Aug 2018 18:47:17 -0600 Subject: [PATCH 51/52] Don't explain what a 400 Bad Request is --- api/server-server/query.yaml | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/api/server-server/query.yaml b/api/server-server/query.yaml index 2f4b8d1e..f569549e 100644 --- a/api/server-server/query.yaml +++ b/api/server-server/query.yaml @@ -49,8 +49,7 @@ paths: description: |- Performs a query to get the mapped room ID and list of resident homeservers in the room for a given room alias. Homeservers should only query room aliases - that belong to the target server (idenfified by the DNS Name in the alias). - The target server may not appear in the resident servers list. + that belong to the target server (identified by the DNS Name in the alias). Servers may wish to cache the response to this query to avoid requesting the information too often. @@ -92,17 +91,6 @@ paths: "another.example.com:8449", ] } - 400: - description: |- - The room alias is not hosted on the server. This can happen if the directory - server is named "example.org" and the room alias ends with "matrix.org". - schema: - $ref: "../client-server/definitions/errors/error.yaml" - examples: - application/json: { - "errcode": "M_UNKNOWN", - "error": "Room alias not hosted on this homeserver." - } 404: description: The room alias was not found. schema: @@ -167,21 +155,6 @@ paths: "displayname": "John Doe", "avatar_url": "mxc://matrix.org/MyC00lAvatar" } - 400: - description: |- - The request was missing parameters or had invalid values for the parameters. This - can happen for: - - * The user not being hosted on the homeserver, - * Lack of a ``user_id`` in the request, or - * The ``field`` requested not being an allowed value. - schema: - $ref: "../client-server/definitions/errors/error.yaml" - examples: - application/json: { - "errcode": "M_INVALID_ARGUMENT_VALUE", - "error": "User is not hosted on this homeserver." - } 404: description: The user does not exist or does not have a profile. schema: From e766606f24820f75820b1ae45c7b1a729d64ddfb Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 2 Aug 2018 18:54:12 -0600 Subject: [PATCH 52/52] join -> leave --- api/server-server/leaving.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server-server/leaving.yaml b/api/server-server/leaving.yaml index 11b0e01d..e287bf58 100644 --- a/api/server-server/leaving.yaml +++ b/api/server-server/leaving.yaml @@ -25,7 +25,7 @@ produces: paths: "/make_leave/{roomId}/{userId}": get: - summary: Get information required to make a join event for a room + summary: Get information required to make a leave event for a room description: |- Asks the receiving server to return information that the sending server will need to prepare a leave event to get out of the room.