diff --git a/api/application-service/definitions/security.yaml b/api/application-service/definitions/security.yaml new file mode 100644 index 00000000..bcfc69c0 --- /dev/null +++ b/api/application-service/definitions/security.yaml @@ -0,0 +1,18 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +homeserverAccessToken: + type: apiKey + description: The ``hs_token`` provided by the application service's registration. + name: access_token + in: query diff --git a/api/application-service/protocols.yaml b/api/application-service/protocols.yaml index e6489cc5..32ac2c3c 100644 --- a/api/application-service/protocols.yaml +++ b/api/application-service/protocols.yaml @@ -19,13 +19,15 @@ host: localhost:8008 schemes: - https - http -basePath: "/" +basePath: /_matrix/app/v1 consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: - "/_matrix/app/unstable/thirdparty/protocol/{protocol}": + "/thirdparty/protocol/{protocol}": get: summary: Retrieve metadata about a specific protocol that the application service supports. description: |- @@ -33,6 +35,8 @@ paths: with specific information about the various third party networks that an application service supports. operationId: getProtocolMetadata + security: + - homeserverAccessToken: [] parameters: - in: path name: protocol @@ -72,7 +76,7 @@ paths: } schema: $ref: ../client-server/definitions/errors/error.yaml - "/_matrix/app/unstable/thirdparty/user/{protocol}": + "/thirdparty/user/{protocol}": get: summary: Retrieve the Matrix User ID of a corresponding third party user. description: |- @@ -80,6 +84,8 @@ paths: User ID linked to a user on the third party network, given a set of user parameters. operationId: queryUserByProtocol + security: + - homeserverAccessToken: [] parameters: - in: path name: protocol @@ -125,12 +131,14 @@ paths: } schema: $ref: ../client-server/definitions/errors/error.yaml - "/_matrix/app/unstable/thirdparty/location/{protocol}": + "/thirdparty/location/{protocol}": get: summary: Retrieve Matrix-side portal rooms leading to a third party location. description: |- Retrieve a list of Matrix portal rooms that lead to the matched third party location. operationId: queryLocationByProtocol + security: + - homeserverAccessToken: [] parameters: - in: path name: protocol @@ -176,13 +184,15 @@ paths: } schema: $ref: ../client-server/definitions/errors/error.yaml - "/_matrix/app/unstable/thirdparty/location": + "/thirdparty/location": get: summary: Reverse-lookup third party locations given a Matrix room alias. description: |- Retrieve an array of third party network locations from a Matrix room alias. operationId: queryLocationByAlias + security: + - homeserverAccessToken: [] parameters: - in: query name: alias @@ -221,12 +231,14 @@ paths: } schema: $ref: ../client-server/definitions/errors/error.yaml - "/_matrix/app/unstable/thirdparty/user": + "/thirdparty/user": get: summary: Reverse-lookup third party users given a Matrix User ID. description: |- Retrieve an array of third party users from a Matrix User ID. operationId: queryUserByID + security: + - homeserverAccessToken: [] parameters: - in: query name: userid diff --git a/api/application-service/query_room.yaml b/api/application-service/query_room.yaml index b885cb86..2fbc87d1 100644 --- a/api/application-service/query_room.yaml +++ b/api/application-service/query_room.yaml @@ -20,11 +20,13 @@ host: localhost:8008 schemes: - https - http -basePath: "/" +basePath: /_matrix/app/v1 consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/rooms/{roomAlias}": get: @@ -36,6 +38,8 @@ paths: homeserver will send this request when it receives a request to join a room alias within the application service's namespace. operationId: queryRoomByAlias + security: + - homeserverAccessToken: [] parameters: - in: path name: roomAlias diff --git a/api/application-service/query_user.yaml b/api/application-service/query_user.yaml index 0431b5e4..da363382 100644 --- a/api/application-service/query_user.yaml +++ b/api/application-service/query_user.yaml @@ -20,11 +20,13 @@ host: localhost:8008 schemes: - https - http -basePath: "/" +basePath: /_matrix/app/v1 consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/users/{userId}": get: @@ -36,6 +38,8 @@ paths: send this request when it receives an event for an unknown user ID in the application service's namespace, such as a room invite. operationId: queryUserById + security: + - homeserverAccessToken: [] parameters: - in: path name: userId diff --git a/api/application-service/transactions.yaml b/api/application-service/transactions.yaml index 8735cc8f..98181196 100644 --- a/api/application-service/transactions.yaml +++ b/api/application-service/transactions.yaml @@ -20,9 +20,11 @@ host: localhost:8008 schemes: - https - http -basePath: "/" +basePath: /_matrix/app/v1 produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/transactions/{txnId}": put: @@ -35,6 +37,8 @@ paths: from message events via the presence of a ``state_key``, rather than via the event type. operationId: sendTransaction + security: + - homeserverAccessToken: [] parameters: - in: path name: txnId diff --git a/api/client-server/room_state.yaml b/api/client-server/room_state.yaml index c04fb803..bda66eb8 100644 --- a/api/client-server/room_state.yaml +++ b/api/client-server/room_state.yaml @@ -87,6 +87,16 @@ paths: type: string description: |- A unique identifier for the event. + 403: + description: |- + The sender doesn't have permission to send the event into the room. + schema: + $ref: "definitions/errors/error.yaml" + examples: + application/json: { + "errcode": "M_FORBIDDEN", + "error": "You do not have permission to send the event." + } tags: - Room participation "/rooms/{roomId}/state/{eventType}": @@ -142,5 +152,15 @@ paths: type: string description: |- A unique identifier for the event. + 403: + description: |- + The sender doesn't have permission to send the event into the room. + schema: + $ref: "definitions/errors/error.yaml" + examples: + application/json: { + "errcode": "M_FORBIDDEN", + "error": "You do not have permission to send the event." + } tags: - Room participation diff --git a/api/client-server/sync.yaml b/api/client-server/sync.yaml index f1997b61..21af0d1e 100644 --- a/api/client-server/sync.yaml +++ b/api/client-server/sync.yaml @@ -77,13 +77,14 @@ paths: - in: query name: set_presence type: string - enum: ["offline"] + enum: ["offline", "online", "unavailable"] description: |- Controls whether the client is automatically marked as online by polling this API. If this parameter is omitted then the client is automatically marked as online when it uses this API. Otherwise if the parameter is set to "offline" then the client is not marked as - being online when it uses this API. + being online when it uses this API. When set to "unavailable", the + client is marked as being idle. x-example: "offline" - in: query name: timeout diff --git a/api/identity/associations.yaml b/api/identity/associations.yaml index 784bb5d6..4225919b 100644 --- a/api/identity/associations.yaml +++ b/api/identity/associations.yaml @@ -46,10 +46,10 @@ paths: description: Validation information for the session. examples: application/json: { - "medium": "email", - "validated_at": 1457622739026, - "address": "louise@bobs.burgers" - } + "medium": "email", + "validated_at": 1457622739026, + "address": "louise@bobs.burgers" + } schema: type: object properties: @@ -62,6 +62,7 @@ paths: validated_at: type: integer description: Timestamp indicating the time that the 3pid was validated. + required: ['medium', 'address', 'validated_at'] 400: description: |- The session has not been validated. @@ -71,16 +72,20 @@ paths: ``errcode`` will be ``M_SESSION_EXPIRED``. examples: application/json: { - "errcode": "M_SESSION_NOT_VALIDATED", - "error": "This validation session has not yet been completed" - } + "errcode": "M_SESSION_NOT_VALIDATED", + "error": "This validation session has not yet been completed" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" 404: description: The Session ID or client secret were not found examples: application/json: { - "errcode": "M_NO_VALID_SESSION", - "error": "No valid session was found matching that sid and client secret" - } + "errcode": "M_NO_VALID_SESSION", + "error": "No valid session was found matching that sid and client secret" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" "/bind": post: summary: Publish an association between a session and a Matrix user ID. @@ -101,10 +106,10 @@ paths: schema: type: object example: { - "sid": "1234", - "client_secret": "monkeys_are_GREAT", - "mxid": "@ears:matrix.org" - } + "sid": "1234", + "client_secret": "monkeys_are_GREAT", + "mxid": "@ears:matrix.org" + } properties: sid: type: string @@ -121,19 +126,19 @@ paths: description: The association was published. examples: application/json: { - "address": "louise@bobs.burgers", - "medium": "email", - "mxid": "@ears:matrix.org", - "not_before": 1428825849161, - "not_after": 4582425849161, - "ts": 1428825849161, + "address": "louise@bobs.burgers", + "medium": "email", + "mxid": "@ears:matrix.org", + "not_before": 1428825849161, + "not_after": 4582425849161, + "ts": 1428825849161, - "signatures": { - "matrix.org": { - "ed25519:0": "ENiU2YORYUJgE6WBMitU0mppbQjidDLanAusj8XS2nVRHPu+0t42OKA/r6zV6i2MzUbNQ3c3MiLScJuSsOiVDQ" - } + "signatures": { + "matrix.org": { + "ed25519:0": "ENiU2YORYUJgE6WBMitU0mppbQjidDLanAusj8XS2nVRHPu+0t42OKA/r6zV6i2MzUbNQ3c3MiLScJuSsOiVDQ" } } + } schema: type: object properties: @@ -158,6 +163,15 @@ paths: signatures: type: object description: The signatures of the verifying identity services which show that the association should be trusted, if you trust the verifying identity services. + $ref: "../../schemas/server-signatures.yaml" + required: + - address + - medium + - mxid + - not_before + - not_after + - ts + - signatures 400: description: |- The association was not published. @@ -167,13 +181,17 @@ paths: ``errcode`` will be ``M_SESSION_EXPIRED``. examples: application/json: { - "errcode": "M_SESSION_NOT_VALIDATED", - "error": "This validation session has not yet been completed" - } + "errcode": "M_SESSION_NOT_VALIDATED", + "error": "This validation session has not yet been completed" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" 404: description: The Session ID or client secret were not found examples: application/json: { - "errcode": "M_NO_VALID_SESSION", - "error": "No valid session was found matching that sid and client secret" - } + "errcode": "M_NO_VALID_SESSION", + "error": "No valid session was found matching that sid and client secret" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" diff --git a/api/identity/email_associations.yaml b/api/identity/email_associations.yaml index 8431c9e8..28f5e680 100644 --- a/api/identity/email_associations.yaml +++ b/api/identity/email_associations.yaml @@ -51,10 +51,10 @@ paths: schema: type: object example: { - "client_secret": "monkeys_are_GREAT", - "email": "foo@example.com", - "send_attempt": 1 - } + "client_secret": "monkeys_are_GREAT", + "email": "foo@example.com", + "send_attempt": 1 + } properties: client_secret: type: string @@ -85,20 +85,28 @@ paths: Session created. examples: application/json: { - "sid": "1234" - } + "sid": "1234" + } schema: type: object properties: sid: type: string description: The session ID. + required: ['sid'] 400: description: | An error ocurred. Some possible errors are: - ``M_INVALID_EMAIL``: The email address provided was invalid. - ``M_EMAIL_SEND_ERROR``: The validation email could not be sent. + examples: + application/json: { + "errcode": "M_INVALID_EMAIL", + "error": "The email address is not valid" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" "/validate/email/submitToken": post: summary: Validate ownership of an email address. @@ -122,10 +130,10 @@ paths: schema: type: object example: { - "sid": "1234", - "client_secret": "monkeys_are_GREAT", - "token": "atoken" - } + "sid": "1234", + "client_secret": "monkeys_are_GREAT", + "token": "atoken" + } properties: sid: type: string @@ -143,14 +151,15 @@ paths: The success of the validation. examples: application/json: { - "success": true - } + "success": true + } schema: type: object properties: success: type: boolean description: Whether the validation was successful or not. + required: ['success'] get: summary: Validate ownership of an email address. description: |- diff --git a/api/identity/invitation_signing.yaml b/api/identity/invitation_signing.yaml index 982dbff7..7de62dd4 100644 --- a/api/identity/invitation_signing.yaml +++ b/api/identity/invitation_signing.yaml @@ -68,9 +68,11 @@ paths: signatures: type: object description: The signature of the mxid, sender, and token. + $ref: "../../schemas/server-signatures.yaml" token: type: string description: The token for the invitation. + required: ['mxid', 'sender', 'signatures', 'token'] examples: application/json: { "mxid": "@foo:bar.com", @@ -84,7 +86,10 @@ paths: } 404: description: Token was not found. - example: { + examples: + application/json: { "errcode": "M_UNRECOGNIZED", "error": "Didn't recognize token" } + schema: + $ref: "../client-server/definitions/errors/error.yaml" diff --git a/api/identity/lookup.yaml b/api/identity/lookup.yaml index bfd2153e..f04436ef 100644 --- a/api/identity/lookup.yaml +++ b/api/identity/lookup.yaml @@ -49,19 +49,18 @@ paths: The association for that 3pid, or the empty object if no association is known. examples: application/json: { - "address": "louise@bobs.burgers", - "medium": "email", - "mxid": "@ears:matrix.org", - "not_before": 1428825849161, - "not_after": 4582425849161, - "ts": 1428825849161, - - "signatures": { - "matrix.org": { - "ed25519:0": "ENiU2YORYUJgE6WBMitU0mppbQjidDLanAusj8XS2nVRHPu+0t42OKA/r6zV6i2MzUbNQ3c3MiLScJuSsOiVDQ" - } + "address": "louise@bobs.burgers", + "medium": "email", + "mxid": "@ears:matrix.org", + "not_before": 1428825849161, + "not_after": 4582425849161, + "ts": 1428825849161, + "signatures": { + "matrix.org": { + "ed25519:0": "ENiU2YORYUJgE6WBMitU0mppbQjidDLanAusj8XS2nVRHPu+0t42OKA/r6zV6i2MzUbNQ3c3MiLScJuSsOiVDQ" } } + } schema: type: object properties: @@ -86,6 +85,15 @@ paths: signatures: type: object description: The signatures of the verifying identity services which show that the association should be trusted, if you trust the verifying identity services. + $ref: "../../schemas/server-signatures.yaml" + required: + - address + - medium + - mxid + - not_before + - not_after + - ts + - signatures "/bulk_lookup": post: summary: Lookup Matrix user IDs for a list of 3pids. @@ -110,9 +118,14 @@ paths: items: type: array title: 3PID mappings + minItems: 2 + maxItems: 2 items: - type: string - title: 3PID medium or address + # TODO: Give real names to these values. Adding a `title` does not work. + #- type: 3PID Medium + #- type: 3PID Address + - type: string + - type: string description: an array of arrays containing the `3PID Types`_ with the ``medium`` in first position and the ``address`` in second position. required: - "threepids" @@ -134,9 +147,16 @@ paths: items: type: array title: 3PID mappings + minItems: 3 + maxItems: 3 items: - type: string - title: 3PID medium or address or the Matrix ID + # TODO: Give real names to these values. Adding a `title` does not work. + #- type: 3PID Medium + #- type: 3PID Address + #- type: Matrix User ID + - type: string + - type: string + - type: string description: an array of array containing the `3PID Types`_ with the ``medium`` in first position, the ``address`` in second position and Matrix ID in third position. required: - "threepids" diff --git a/api/identity/phone_associations.yaml b/api/identity/phone_associations.yaml index c2cc6cfe..f6b1bd45 100644 --- a/api/identity/phone_associations.yaml +++ b/api/identity/phone_associations.yaml @@ -51,11 +51,11 @@ paths: schema: type: object example: { - "client_secret": "monkeys_are_GREAT", - "country": "GB", - "phone_number": "07700900001", - "send_attempt": 1 - } + "client_secret": "monkeys_are_GREAT", + "country": "GB", + "phone_number": "07700900001", + "send_attempt": 1 + } properties: client_secret: type: string @@ -91,20 +91,28 @@ paths: Session created. examples: application/json: { - "sid": "1234" - } + "sid": "1234" + } schema: type: object properties: sid: type: string description: The session ID. + required: ['sid'] 400: description: | An error ocurred. Some possible errors are: - ``M_INVALID_ADDRESS``: The phone number provided was invalid. - ``M_SEND_ERROR``: The validation SMS could not be sent. + examples: + application/json: { + "errcode": "M_INVALID_ADDRESS", + "error": "The phone number is not valid" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" "/validate/msisdn/submitToken": post: summary: Validate ownership of a phone number. @@ -128,10 +136,10 @@ paths: schema: type: object example: { - "sid": "1234", - "client_secret": "monkeys_are_GREAT", - "token": "atoken" - } + "sid": "1234", + "client_secret": "monkeys_are_GREAT", + "token": "atoken" + } properties: sid: type: string @@ -149,14 +157,15 @@ paths: The success of the validation. examples: application/json: { - "success": true - } + "success": true + } schema: type: object properties: success: type: boolean description: Whether the validation was successful or not. + required: ['success'] get: summary: Validate ownership of a phone number. description: |- diff --git a/api/identity/pubkey.yaml b/api/identity/pubkey.yaml index 00796975..9cb7c74e 100644 --- a/api/identity/pubkey.yaml +++ b/api/identity/pubkey.yaml @@ -45,13 +45,25 @@ paths: The public key exists. examples: application/json: { - "public_key": "VXuGitF39UH5iRfvbIknlvlAVKgD1BsLDMvBf0pmp7c" - } + "public_key": "VXuGitF39UH5iRfvbIknlvlAVKgD1BsLDMvBf0pmp7c" + } schema: type: object properties: public_key: type: string + description: Unpadded Base64 encoded public key. + required: ['public_key'] + 404: + description: + The public key was not found. + examples: + application/json: { + "errcode": "M_NOT_FOUND", + "error": "The public key was not found" + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" "/pubkey/isvalid": get: summary: Check whether a long-term public key is valid. @@ -72,14 +84,15 @@ paths: The validity of the public key. examples: application/json: { - "valid": true - } + "valid": true + } schema: type: object properties: valid: type: boolean description: Whether the public key is recognised and is currently valid. + required: ['valid'] "/pubkey/ephemeral/isvalid": get: summary: Check whether a short-term public key is valid. @@ -108,3 +121,4 @@ paths: valid: type: boolean description: Whether the public key is recognised and is currently valid. + required: ['valid'] diff --git a/api/identity/store_invite.yaml b/api/identity/store_invite.yaml index 6b847b5b..1eca7198 100644 --- a/api/identity/store_invite.yaml +++ b/api/identity/store_invite.yaml @@ -54,11 +54,11 @@ paths: schema: type: object example: { - "medium": "email", - "address": "foo@bar.baz", - "room_id": "!something:example.tld", - "sender": "@bob:example.com" - } + "medium": "email", + "address": "foo@bar.baz", + "room_id": "!something:example.tld", + "sender": "@bob:example.com" + } properties: medium: type: string @@ -90,15 +90,16 @@ paths: display_name: type: string description: The generated (redacted) display_name. + required: ['token', 'public_keys', 'display_name'] example: application/json: { - "token": "sometoken", - "public_keys": [ - "serverpublickey", - "ephemeralpublickey" - ], - "display_name": "f...@b..." - } + "token": "sometoken", + "public_keys": [ + "serverpublickey", + "ephemeralpublickey" + ], + "display_name": "f...@b..." + } 400: description: | An error has occured. @@ -108,7 +109,9 @@ paths: error code will be ``M_UNRECOGNIZED``. examples: application/json: { - "errcode": "M_THREEPID_IN_USE", - "error": "Binding already known", - "mxid": mxid - } + "errcode": "M_THREEPID_IN_USE", + "error": "Binding already known", + "mxid": mxid + } + schema: + $ref: "../client-server/definitions/errors/error.yaml" diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml index cf3988a2..c23163d7 100644 --- a/api/server-server/events.yaml +++ b/api/server-server/events.yaml @@ -49,7 +49,8 @@ paths: responses: 200: description: |- - The fully resolved state for the room, including the authorization + The fully resolved state for the room, prior to considering any state + changes induced by the requested event. Includes the authorization chain for the events. schema: type: object @@ -96,7 +97,8 @@ paths: responses: 200: description: |- - The fully resolved state for the room, including the authorization + The fully resolved state for the room, prior to considering any state + changes induced by the requested event. Includes the authorization chain for the events. schema: type: object diff --git a/changelogs/application_service.rst b/changelogs/application_service.rst new file mode 100644 index 00000000..e69de29b diff --git a/changelogs/application_service/newsfragments/.gitignore b/changelogs/application_service/newsfragments/.gitignore new file mode 100644 index 00000000..b722e9e1 --- /dev/null +++ b/changelogs/application_service/newsfragments/.gitignore @@ -0,0 +1 @@ +!.gitignore \ No newline at end of file diff --git a/changelogs/application_service/pyproject.toml b/changelogs/application_service/pyproject.toml new file mode 100644 index 00000000..44d430e8 --- /dev/null +++ b/changelogs/application_service/pyproject.toml @@ -0,0 +1,30 @@ +[tool.towncrier] + filename = "../application_service.rst" + directory = "newsfragments" + issue_format = "`#{issue} `_" + title_format = "{version}" + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "new" + name = "New Endpoints" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Backwards Compatible Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "clarification" + name = "Spec Clarifications" + showcontent = true diff --git a/changelogs/client_server/newsfragments/1176.new b/changelogs/client_server/newsfragments/1176.new new file mode 100644 index 00000000..41e30799 --- /dev/null +++ b/changelogs/client_server/newsfragments/1176.new @@ -0,0 +1 @@ +Specify how to control the power level required for ``@room`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1465.feature b/changelogs/client_server/newsfragments/1465.feature new file mode 100644 index 00000000..649cf222 --- /dev/null +++ b/changelogs/client_server/newsfragments/1465.feature @@ -0,0 +1 @@ +Share room decryption keys between devices diff --git a/changelogs/client_server/newsfragments/1590.clarification b/changelogs/client_server/newsfragments/1590.clarification new file mode 100644 index 00000000..27999193 --- /dev/null +++ b/changelogs/client_server/newsfragments/1590.clarification @@ -0,0 +1 @@ +Document the 403 error for sending state events. diff --git a/changelogs/client_server/newsfragments/1596.clarification b/changelogs/client_server/newsfragments/1596.clarification new file mode 100644 index 00000000..3dde069f --- /dev/null +++ b/changelogs/client_server/newsfragments/1596.clarification @@ -0,0 +1 @@ +specify how to handle multiple olm sessions with the same device \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/780.feature b/changelogs/client_server/newsfragments/780.feature new file mode 100644 index 00000000..74725754 --- /dev/null +++ b/changelogs/client_server/newsfragments/780.feature @@ -0,0 +1 @@ +Add more presence options to the ``set_presence`` parameter of ``/sync``. (Thanks @mujx!) diff --git a/changelogs/push_gateway.rst b/changelogs/push_gateway.rst new file mode 100644 index 00000000..33a7683c --- /dev/null +++ b/changelogs/push_gateway.rst @@ -0,0 +1,6 @@ +r0.1.0 +====== + +The first release of the Push Gateway specification. This release contains +a single endpoint, ``/notify``, that pushers may use to send push notifications +to clients. diff --git a/changelogs/push_gateway/newsfragments/.gitignore b/changelogs/push_gateway/newsfragments/.gitignore new file mode 100644 index 00000000..b722e9e1 --- /dev/null +++ b/changelogs/push_gateway/newsfragments/.gitignore @@ -0,0 +1 @@ +!.gitignore \ No newline at end of file diff --git a/changelogs/push_gateway/pyproject.toml b/changelogs/push_gateway/pyproject.toml new file mode 100644 index 00000000..dad1bc04 --- /dev/null +++ b/changelogs/push_gateway/pyproject.toml @@ -0,0 +1,30 @@ +[tool.towncrier] + filename = "../push_gateway.rst" + directory = "newsfragments" + issue_format = "`#{issue} `_" + title_format = "{version}" + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "new" + name = "New Endpoints" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Backwards Compatible Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "clarification" + name = "Spec Clarifications" + showcontent = true diff --git a/event-schemas/examples/m.forwarded_room_key b/event-schemas/examples/m.forwarded_room_key new file mode 100644 index 00000000..ef1d6180 --- /dev/null +++ b/event-schemas/examples/m.forwarded_room_key @@ -0,0 +1,14 @@ +{ + "content": { + "algorithm": "m.megolm.v1.aes-sha2", + "room_id": "!Cuyf34gef24t:localhost", + "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ", + "session_key": "AgAAAADxKHa9uFxcXzwYoNueL5Xqi69IkD4sni8Llf...", + "sender_key": "RF3s+E7RkTQTGF2d8Deol0FkQvgII2aJDf3/Jp5mxVU", + "sender_claimed_ed25519_key": "aj40p+aw64yPIdsxoog8jhPu9i7l7NcFRecuOQblE3Y", + "forwarding_curve25519_key_chain": [ + "hPQNcabIABgGnx3/ACv/jmMmiQHoeFfuLB17tzWp6Hw" + ] + }, + "type": "m.forwarded_room_key" +} diff --git a/event-schemas/examples/m.room.power_levels b/event-schemas/examples/m.room.power_levels index 0c8f8bc5..2b0aaca5 100644 --- a/event-schemas/examples/m.room.power_levels +++ b/event-schemas/examples/m.room.power_levels @@ -14,7 +14,10 @@ "users": { "@example:localhost": 100 }, - "users_default": 0 + "users_default": 0, + "notifications": { + "room": 20 + } }, "state_key": "", "origin_server_ts": 1431961217939, diff --git a/event-schemas/examples/m.room_key_request#cancel_request b/event-schemas/examples/m.room_key_request#cancel_request new file mode 100644 index 00000000..c6eb25de --- /dev/null +++ b/event-schemas/examples/m.room_key_request#cancel_request @@ -0,0 +1,8 @@ +{ + "content": { + "action": "cancel_request", + "requesting_device_id": "RJYKSTBOIE", + "request_id": "1495474790150.19" + }, + "type": "m.room_key_request" +} diff --git a/event-schemas/examples/m.room_key_request#request b/event-schemas/examples/m.room_key_request#request new file mode 100644 index 00000000..8557f08e --- /dev/null +++ b/event-schemas/examples/m.room_key_request#request @@ -0,0 +1,14 @@ +{ + "content": { + "body": { + "algorithm": "m.megolm.v1.aes-sha2", + "room_id": "!Cuyf34gef24t:localhost", + "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ", + "sender_key": "RF3s+E7RkTQTGF2d8Deol0FkQvgII2aJDf3/Jp5mxVU" + }, + "action": "request", + "requesting_device_id": "RJYKSTBOIE", + "request_id": "1495474790150.19" + }, + "type": "m.room_key_request" +} diff --git a/event-schemas/schema/m.forwarded_room_key b/event-schemas/schema/m.forwarded_room_key new file mode 100644 index 00000000..f0beed2b --- /dev/null +++ b/event-schemas/schema/m.forwarded_room_key @@ -0,0 +1,59 @@ +--- +allOf: + - $ref: core-event-schema/event.yaml + +description: |- + This event type is used to forward keys for end-to-end encryption. Typically + it is encrypted as an ``m.room.encrypted`` event, then sent as a `to-device`_ + event. +properties: + content: + properties: + algorithm: + type: string + description: |- + The encryption algorithm the key in this event is to be used with. + room_id: + type: string + description: The room where the key is used. + sender_key: + type: string + description: |- + The Curve25519 key of the device which initiated the session originally. + session_id: + type: string + description: The ID of the session that the key is for. + session_key: + type: string + description: The key to be exchanged. + sender_claimed_ed25519_key: + type: string + description: |- + The Ed25519 key of the device which initiated the session originally. + It is 'claimed' because the receiving device has no way to tell that the + original room_key actually came from a device which owns the private part of + this key unless they have done device verification. + forwarding_curve25519_key_chain: + type: array + items: + type: string + description: |- + Chain of Curve25519 keys. It starts out empty, but each time the + key is forwarded to another device, the previous sender in the chain is added + to the end of the list. For example, if the key is forwarded from A to B to + C, this field is empty between A and B, and contains A's Curve25519 key between + B and C. + required: + - algorithm + - room_id + - session_id + - session_key + - sender_claimed_ed25519_key + - forwarding_curve25519_key_chain + - sender_key + type: object + type: + enum: + - m.forwarded_room_key + type: string +type: object diff --git a/event-schemas/schema/m.room.power_levels b/event-schemas/schema/m.room.power_levels index b00d86a9..9bb12993 100644 --- a/event-schemas/schema/m.room.power_levels +++ b/event-schemas/schema/m.room.power_levels @@ -85,6 +85,18 @@ properties: ``user_id`` is mentioned in the ``users`` key. Defaults to 0 if unspecified. type: integer + notifications: + properties: + room: + type: integer + description: The level required to trigger an ``@room`` notification. Defaults to 50 if unspecified. + additionalProperties: + type: integer + description: |- + The power level requirements for specific notification types. + This is a mapping from ``key`` to power level for that notifications key. + title: Notifications + type: object type: object state_key: description: A zero-length string. diff --git a/event-schemas/schema/m.room_key_request b/event-schemas/schema/m.room_key_request new file mode 100644 index 00000000..007d0086 --- /dev/null +++ b/event-schemas/schema/m.room_key_request @@ -0,0 +1,61 @@ +--- +allOf: + - $ref: core-event-schema/event.yaml + +description: |- + This event type is used to request keys for end-to-end encryption. It is sent as an + unencrypted `to-device`_ event. +properties: + content: + properties: + body: + description: |- + Information about the requested key. Required when ``action`` is + ``request``. + properties: + algorithm: + type: string + description: |- + The encryption algorithm the requested key in this event is to be used + with. + room_id: + type: string + description: The room where the key is used. + sender_key: + type: string + description: |- + The Curve25519 key of the device which initiated the session originally. + session_id: + type: string + description: The ID of the session that the key is for. + required: + - algorithm + - room_id + - session_id + - sender_key + type: object + title: RequestedKeyInfo + action: + enum: + - request + - cancel_request + type: string + requesting_device_id: + description: ID of the device requesting the key. + type: string + request_id: + description: |- + A random string uniquely identifying the request for a key. If the key is + requested multiple times, it should be reused. It should also reused in order + to cancel a request. + type: string + required: + - action + - requesting_device_id + - request_id + type: object + type: + enum: + - m.room_key_request + type: string +type: object diff --git a/meta/releasing_a_spec.md b/meta/releasing_a_spec.md new file mode 100644 index 00000000..ac3d21fa --- /dev/null +++ b/meta/releasing_a_spec.md @@ -0,0 +1,48 @@ +# How to release a specification + +There are several specifications that belong to matrix, such as the client-server +specification, server-server specification, and identity server specification. Each +of these gets released independently of each other with their own version numbers. + +Once a specification is ready for release, a branch should be created to track the +changes in and to hold potential future hotfixes. This should be the name of the +specification (as it appears in the directory structure of this project) followed +by "release-" and the release version. For example, if the Client-Server Specification +was getting an r0.4.0 release, the branch name would be `client_server/release-r0.4.0`. + +*Note*: Historical releases prior to this process may or may not have an appropriate +release branch. Releases after this document came into place will have an appropriate +branch. + +The remainder of the process is as follows: +1. Activate your Python 3 virtual environment. +1. Having checked out the new release branch, navigate your way over to `./changelogs`. +1. Follow the release instructions provided in the README.md located there. +1. Update the changelog section of the specification you're releasing to make a reference + to the new version. +1. Update any version/link references across all specifications. +1. Ensure the `targets.yml` file lists the version correctly. +1. Commit the changes and PR them to master. +1. Tag the release with the format `client_server/r0.4.0`. +1. Add the changes to the matrix-org/matrix.org repository (for historic tracking). + * This is done by making a PR to the `unstyled_docs/spec` folder for the version and + specification you're releasing. + * Don't forget to symlink the new release as `latest`. +1. Perform a release on GitHub to tag the release. +1. Yell from the mountaintop to the world about the new release. + +### Creating a release for a brand-new specification + +Some specifications may not have ever had a release, and therefore need a bit more work +to become ready. + +1. Activate your Python 3 virtual environment. +1. Having checked out the new release branch, navigate your way over to `./changelogs`. +1. Follow the "new changelog" instructions provided in the README.md located there. +1. Open the specification RST file and make some changes: + * Using a released specification as a template, update the changelog section. + * Use the appropriate changelog variable in the RST. +1. Create/define the appropriate variables in `gendoc.py`. +1. Update `targets.yml`. +1. Update any version/link references across all specifications. +1. Follow the regular release process. diff --git a/schemas/server-signatures.yaml b/schemas/server-signatures.yaml new file mode 100644 index 00000000..a1855256 --- /dev/null +++ b/schemas/server-signatures.yaml @@ -0,0 +1,24 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +example: { + "example.com": { + "ed25519:0": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus" + } +} +additionalProperties: + type: object + title: Server Signatures + additionalProperties: + type: string \ No newline at end of file diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 16c40af5..ee3ef599 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -154,7 +154,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): # string are file paths to RST blobs if isinstance(file_info, str): log("%s %s" % (">" * (1 + title_level), file_info)) - with open(os.path.join(spec_dir, file_info), "r") as f: + with open(os.path.join(spec_dir, file_info), "r", encoding="utf-8") as f: rst = None if adjust_titles: rst = load_with_adjusted_titles( @@ -186,7 +186,7 @@ def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): def build_spec(target, out_filename): log("Building templated file %s" % out_filename) - with open(out_filename, "wb") as outfile: + with open(out_filename, "w", encoding="utf-8") as outfile: for file_info in target["files"]: section = get_rst( file_info=file_info, @@ -195,7 +195,7 @@ def build_spec(target, out_filename): spec_dir=spec_dir, adjust_titles=True ) - outfile.write(section.encode('UTF-8')) + outfile.write(section) """ @@ -223,8 +223,8 @@ def fix_relative_titles(target, filename, out_filename): "^[" + re.escape("".join(title_styles)) + "]{3,}$" ) current_title_style = None - with open(filename, "r") as infile: - with open(out_filename, "w") as outfile: + with open(filename, "r", encoding="utf-8") as infile: + with open(out_filename, "w", encoding="utf-8") as outfile: for line in infile.readlines(): if not relative_title_matcher.match(line): if title_matcher.match(line): @@ -263,8 +263,8 @@ def fix_relative_titles(target, filename, out_filename): def rst2html(i, o, stylesheets): log("rst2html %s -> %s" % (i, o)) - with open(i, "r") as in_file: - with open(o, "w") as out_file: + with open(i, "r", encoding="utf-8") as in_file: + with open(o, "w", encoding="utf-8") as out_file: publish_file( source=in_file, destination=out_file, @@ -280,16 +280,15 @@ def rst2html(i, o, stylesheets): def addAnchors(path): log("add anchors %s" % path) - with open(path, "rb") as f: + with open(path, "r", encoding="utf-8") as f: lines = f.readlines() replacement = r'

\n\1' - with open(path, "wb") as f: + with open(path, "w", encoding="utf-8") as f: for line in lines: - line = line.decode("UTF-8") line = re.sub(r'()', replacement, line.rstrip()) line = re.sub(r'(
)', replacement, line.rstrip()) - f.write((line + "\n").encode('UTF-8')) + f.write(line + "\n") def run_through_template(input_files, set_verbose, substitutions): @@ -518,6 +517,14 @@ if __name__ == '__main__': "--server_release", "-s", action="store", default="unstable", help="The server-server release tag to generate, e.g. r1.2" ) + parser.add_argument( + "--appservice_release", "-a", action="store", default="unstable", + help="The appservice release tag to generate, e.g. r1.2" + ) + parser.add_argument( + "--push_gateway_release", "-p", action="store", default="unstable", + help="The push gateway release tag to generate, e.g. r1.2" + ) parser.add_argument( "--list_targets", action="store_true", help="Do not update the specification. Instead print a list of targets.", @@ -542,6 +549,8 @@ if __name__ == '__main__': "%CLIENT_MAJOR_VERSION%": "r0", "%SERVER_RELEASE_LABEL%": args.server_release, "%SERVER_MAJOR_VERSION%": extract_major(args.server_release), + "%APPSERVICE_RELEASE_LABEL%": args.appservice_release, + "%PUSH_GATEWAY_RELEASE_LABEL%": args.push_gateway_release, } exit (main(args.target or ["all"], args.dest, args.nodelete, substitutions)) diff --git a/scripts/generate-matrix-org-assets b/scripts/generate-matrix-org-assets index ed08f81d..76032850 100755 --- a/scripts/generate-matrix-org-assets +++ b/scripts/generate-matrix-org-assets @@ -8,8 +8,11 @@ cd `dirname $0`/.. mkdir -p assets -# generate specification/proposals.rst -./scripts/proposals.py +if [ "$CIRCLECI" != "true" ] +then + # generate specification/proposals.rst + ./scripts/proposals.py +fi # generate the spec docs ./scripts/gendoc.py -d assets/spec diff --git a/scripts/templating/matrix_templates/sections.py b/scripts/templating/matrix_templates/sections.py index 1a93c723..740e6b2e 100644 --- a/scripts/templating/matrix_templates/sections.py +++ b/scripts/templating/matrix_templates/sections.py @@ -31,6 +31,15 @@ class MatrixSections(Sections): def render_client_server_changelog(self): changelogs = self.units.get("changelogs") return changelogs["client_server"] + + # TODO: We should make this a generic variable instead of having to add functions all the time. + def render_push_gateway_changelog(self): + changelogs = self.units.get("changelogs") + return changelogs["push_gateway"] + + def render_application_service_changelog(self): + changelogs = self.units.get("changelogs") + return changelogs["application_service"] def _render_events(self, filterFn, sortFn): template = self.env.get_template("events.tmpl") diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 81da3f6a..427b3a00 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -125,7 +125,7 @@ def resolve_references(path, schema): if '$ref' in schema: value = schema['$ref'] path = os.path.join(os.path.dirname(path), value) - with open(path) as f: + with open(path, encoding="utf-8") as f: ref = yaml.load(f, OrderedLoader) result = resolve_references(path, ref) del schema['$ref'] @@ -664,11 +664,11 @@ class MatrixUnits(Units): continue filepath = os.path.join(path, filename) logger.info("Reading swagger API: %s" % filepath) - with open(filepath, "r") as f: + with open(filepath, "r", encoding="utf-8") as f: # strip .yaml group_name = filename[:-5].replace("-", "_") group_name = "%s_%s" % (group_name, suffix) - api = yaml.load(f.read(), OrderedLoader) + api = yaml.load(f, OrderedLoader) api = resolve_references(filepath, api) api["__meta"] = self._load_swagger_meta( api, group_name @@ -698,11 +698,11 @@ class MatrixUnits(Units): continue filepath = os.path.join(path, filename) logger.info("Reading swagger definition: %s" % filepath) - with open(filepath, "r") as f: + with open(filepath, "r", encoding="utf-8") as f: # strip .yaml group_name = re.sub(r"[^a-zA-Z0-9_]", "_", filename[:-5]) group_name = "%s_%s" % (prefix, group_name) - definition = yaml.load(f.read(), OrderedLoader) + definition = yaml.load(f, OrderedLoader) definition = resolve_references(filepath, definition) if 'type' not in definition: continue @@ -741,7 +741,7 @@ class MatrixUnits(Units): event_type = filename[:-5] # strip the ".yaml" logger.info("Reading event schema: %s" % filepath) - with open(filepath) as f: + with open(filepath, encoding="utf-8") as f: event_schema = yaml.load(f, OrderedLoader) schema_info = process_data_type( @@ -754,6 +754,8 @@ class MatrixUnits(Units): def load_apis(self, substitutions): cs_ver = substitutions.get("%CLIENT_RELEASE_LABEL%", "unstable") fed_ver = substitutions.get("%SERVER_RELEASE_LABEL%", "unstable") + as_ver = substitutions.get("%APPSERVICE_RELEASE_LABEL%", "unstable") + push_gw_ver = substitutions.get("%PUSH_GATEWAY_RELEASE_LABEL%", "unstable") # we abuse the typetable to return this info to the templates return TypeTable(rows=[ @@ -766,16 +768,16 @@ class MatrixUnits(Units): fed_ver, "Federation between servers", ), TypeTableRow( - "`Application Service API `_", - "unstable", + "`Application Service API `_", + as_ver, "Privileged server plugins", ), TypeTableRow( "`Identity Service API `_", "unstable", "Mapping of third party IDs to Matrix IDs", ), TypeTableRow( - "`Push Gateway API `_", - "unstable", + "`Push Gateway API `_", + push_gw_ver, "Push notifications for Matrix events", ), ]) @@ -791,7 +793,7 @@ class MatrixUnits(Units): filepath = os.path.join(path, filename) logger.info("Reading event example: %s" % filepath) try: - with open(filepath, "r") as f: + with open(filepath, "r", encoding="utf-8") as f: example = json.load(f) examples[filename] = examples.get(filename, []) examples[filename].append(example) @@ -829,7 +831,7 @@ class MatrixUnits(Units): def read_event_schema(self, filepath): logger.info("Reading %s" % filepath) - with open(filepath, "r") as f: + with open(filepath, "r", encoding="utf-8") as f: json_schema = yaml.load(f, OrderedLoader) schema = { @@ -941,7 +943,7 @@ class MatrixUnits(Units): title_part = None changelog_lines = [] - with open(path, "r") as f: + with open(path, "r", encoding="utf-8") as f: lines = f.readlines() prev_line = None for line in (tc_lines + lines): diff --git a/specification/appendices/identifier_grammar.rst b/specification/appendices/identifier_grammar.rst index e72fdada..92071611 100644 --- a/specification/appendices/identifier_grammar.rst +++ b/specification/appendices/identifier_grammar.rst @@ -23,13 +23,37 @@ A homeserver is uniquely identified by its server name. This value is used in a number of identifiers, as described below. The server name represents the address at which the homeserver in question can -be reached by other homeservers. The complete grammar is:: +be reached by other homeservers. All valid server names are included by the +following grammar:: - server_name = host [ ":" port] - port = *DIGIT + server_name = hostname [ ":" port ] -where ``host`` is as defined by `RFC3986, section 3.2.2 -`_. + port = *DIGIT + + hostname = IPv4address / "[" IPv6address "]" / dns-name + + IPv4address = 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT + + IPv6address = 2*45IPv6char + + IPv6char = DIGIT / %x41-46 / %x61-66 / ":" / "." + ; 0-9, A-F, a-f, :, . + + dns-name = *255dns-char + + dns-char = DIGIT / ALPHA / "-" / "." + +— in other words, the server name is the hostname, followed by an optional +numeric port specifier. The hostname may be a dotted-quad IPv4 address literal, +an IPv6 address literal surrounded with square brackets, or a DNS name. + +IPv4 literals must be a sequence of four decimal numbers in the +range 0 to 255, separated by ``.``. IPv6 literals must be as specified by +`RFC3513, section 2.2 `_. + +DNS names for use with Matrix should follow the conventional restrictions for +internet hostnames: they should consist of a series of labels separated by +``.``, where each label consists of the alphanumeric characters or hyphens. Examples of valid server names are: @@ -40,6 +64,20 @@ Examples of valid server names are: * ``[1234:5678::abcd]`` (IPv6 literal) * ``[1234:5678::abcd]:5678`` (IPv6 literal with explicit port) +.. Note:: + + This grammar is based on the standard for internet host names, as specified + by `RFC1123, section 2.1 `_, + with an extension for IPv6 literals. + +Server names must be treated case-sensitively: in other words, +``@user:matrix.org`` is a different person from ``@user:MATRIX.ORG``. + +Some recommendations for a choice of server name follow: + +* The length of the complete server name should not exceed 230 characters. +* Server names should not use upper-case characters. + Room Versions ~~~~~~~~~~~~~ @@ -51,7 +89,7 @@ not understanding the new rules. A room version is defined as a string of characters which MUST NOT exceed 32 codepoints in length. Room versions MUST NOT be empty and SHOULD contain only -the characters ``a-z``, ``0-9``, ``.``, and ``-``. +the characters ``a-z``, ``0-9``, ``.``, and ``-``. Room versions are not intended to be parsed and should be treated as opaque identifiers. Room versions consisting only of the characters ``0-9`` and ``.`` diff --git a/specification/application_service_api.rst b/specification/application_service_api.rst index 5b7abf4a..69d39d21 100644 --- a/specification/application_service_api.rst +++ b/specification/application_service_api.rst @@ -30,22 +30,37 @@ irrespective of the underlying homeserver implementation. .. contents:: Table of Contents .. sectnum:: -Specification version ---------------------- +Changelog +--------- + + +.. topic:: Version: unstable +{{application_service_changelog}} This version of the specification is generated from `matrix-doc `_ as of Git commit `{{git_version}} `_. +For the full historical changelog, see +https://github.com/matrix-org/matrix-doc/blob/master/changelogs/application_service.rst + +Other versions of this specification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following other versions are also available, in reverse chronological order: + +- `HEAD `_: Includes all changes since the latest versioned release. + + Application Services -------------------- -Application services are passive and can only observe events from a given -homeserver (HS). They can inject events into rooms they are participating in. +Application services are passive and can only observe events from homeserver. +They can inject events into rooms they are participating in. They cannot prevent events from being sent, nor can they modify the content of the event being sent. In order to observe events from a homeserver, the homeserver needs to be configured to pass certain types of traffic to the application service. This is achieved by manually configuring the homeserver -with information about the application service (AS). +with information about the application service. Registration ~~~~~~~~~~~~ @@ -169,6 +184,34 @@ An example registration file for an IRC-bridging application service is below: Homeserver -> Application Service API ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Legacy routes ++++++++++++++ + +Previous drafts of the application service specification had a mix of endpoints +that have been used in the wild for a significant amount of time. The application +service specification now defines a version on all endpoints to be more compatible +with the rest of the Matrix specification and the future. + +Homeservers should attempt to use the specified endpoints first when communicating +with application services. However, if the application service receives an http status +code that does not indicate success (ie: 404, 500, 501, etc) then the homeserver +should fall back to the older endpoints for the application service. + +The older endpoints have the exact same request body and response format, they +just belong at a different path. The equivalent path for each is as follows: + +* ``/_matrix/app/v1/transactions/{txnId}`` becomes ``/transactions/{txnId}`` +* ``/_matrix/app/v1/users/{userId}`` becomes ``/users/{userId}`` +* ``/_matrix/app/v1/rooms/{roomAlias}`` becomes ``/rooms/{roomAlias}`` +* ``/_matrix/app/v1/thirdparty/protocol/{protocol}`` becomes ``/_matrix/app/unstable/thirdparty/protocol/{protocol}`` +* ``/_matrix/app/v1/thirdparty/user/{user}`` becomes ``/_matrix/app/unstable/thirdparty/user/{user}`` +* ``/_matrix/app/v1/thirdparty/location/{location}`` becomes ``/_matrix/app/unstable/thirdparty/location/{location}`` +* ``/_matrix/app/v1/thirdparty/user`` becomes ``/_matrix/app/unstable/thirdparty/user`` +* ``/_matrix/app/v1/thirdparty/location`` becomes ``/_matrix/app/unstable/thirdparty/location`` + +Homeservers should periodically try again for the newer endpoints because the +application service may have been updated. + Pushing events ++++++++++++++ @@ -179,24 +222,24 @@ events. Each list of events includes a transaction ID, which works as follows: Typical HS ---> AS : Homeserver sends events with transaction ID T. - <--- : AS sends back 200 OK. + <--- : Application Service sends back 200 OK. AS ACK Lost HS ---> AS : Homeserver sends events with transaction ID T. <-/- : AS 200 OK is lost. HS ---> AS : Homeserver retries with the same transaction ID of T. - <--- : AS sends back 200 OK. If the AS had processed these events - already, it can NO-OP this request (and it knows if it is the same - events based on the transaction ID). + <--- : Application Service sends back 200 OK. If the AS had processed these + events already, it can NO-OP this request (and it knows if it is the + same events based on the transaction ID). The events sent to the application service should be linearised, as if they were from the event stream. The homeserver MUST maintain a queue of transactions to -send to the AS. If the application service cannot be reached, the homeserver -SHOULD backoff exponentially until the application service is reachable again. +send to the application service. If the application service cannot be reached, the +homeserver SHOULD backoff exponentially until the application service is reachable again. As application services cannot *modify* the events in any way, these requests can be made without blocking other aspects of the homeserver. Homeservers MUST NOT alter (e.g. add more) events they were going to send within that transaction ID -on retries, as the AS may have already processed the events. +on retries, as the application service may have already processed the events. {{transactions_as_http_api}} @@ -337,7 +380,7 @@ users needs API changes in order to: - Have a 'passwordless' user. This involves bypassing the registration flows entirely. This is achieved by -including the AS token on a ``/register`` request, along with a login type of +including the ``as_token`` on a ``/register`` request, along with a login type of ``m.login.application_service`` to set the desired user ID without a password. :: @@ -374,13 +417,18 @@ additional parameters on the ``/publicRooms`` client-server endpoint. {{appservice_room_directory_cs_http_api}} -Event fields -~~~~~~~~~~~~ +Referencing messages from a third party network +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. TODO-TravisR: Fix this section to be a general "3rd party networks" section +Application services should include an ``external_url`` in the ``content`` of +events it emits to indicate where the message came from. This typically applies +to application services that bridge other networks into Matrix, such as IRC, +where an HTTP URL may be available to reference. -We recommend that any events that originated from a remote network should -include an ``external_url`` field in their content to provide a way for Matrix -clients to link into the 'native' client from which the event originated. -For instance, this could contain the message-ID for emails/nntp posts, or a link -to a blog comment when bridging blog comment traffic in & out of Matrix. +Clients should provide users with a way to access the ``external_url`` if it +is present. Clients should additionally ensure the URL has a scheme of ``https`` +or ``http`` before making use of it. + +The presence of an ``external_url`` on an event does not necessarily mean the +event was sent from an application service. Clients should be wary of the URL +contained within, as it may not be a legitimate reference to the event's source. diff --git a/specification/identity_service_api.rst b/specification/identity_service_api.rst index 3b037caf..7bbd8ae8 100644 --- a/specification/identity_service_api.rst +++ b/specification/identity_service_api.rst @@ -23,7 +23,7 @@ user identifiers. From time to time, it is useful to refer to users by other number. This identity service specification describes how mappings between third-party identifiers and Matrix user identifiers can be established, validated, and used. This description technically may apply to any 3pid, but in -practice has only been applied specifically to email addresses. +practice has only been applied specifically to email addresses and phone numbers. .. contents:: Table of Contents .. sectnum:: @@ -56,6 +56,75 @@ is left as an exercise for the client. 3PID types are described in `3PID Types`_ Appendix. +API Standards +------------- + +The mandatory baseline for identity service communication in Matrix is exchanging +JSON objects over HTTP APIs. HTTPS is required for communication, and all API calls +use a Content-Type of ``application/json``. In addition, strings MUST be encoded as +UTF-8. + +Any errors which occur at the Matrix API level MUST return a "standard error response". +This is a JSON object which looks like: + +.. code:: json + + { + "errcode": "", + "error": "" + } + +The ``error`` string will be a human-readable error message, usually a sentence +explaining what went wrong. The ``errcode`` string will be a unique string +which can be used to handle an error message e.g. ``M_FORBIDDEN``. There may be +additional keys depending on the error, but the keys ``error`` and ``errcode`` +MUST always be present. + +Some standard error codes are below: + +:``M_NOT_FOUND``: + The resource requested could not be located. + +:``M_MISSING_PARAMS``: + The request was missing one or more parameters. + +:``M_INVALID_PARAM``: + The request contained one or more invalid parameters. + +:``M_SESSION_NOT_VALIDATED``: + The session has not been validated. + +:``M_NO_VALID_SESSION``: + A session could not be located for the given parameters. + +:``M_SESSION_EXPIRED``: + The session has expired and must be renewed. + +:``M_INVALID_EMAIL``: + The email address provided was not valid. + +:``M_EMAIL_SEND_ERROR``: + There was an error sending an email. Typically seen when attempting to verify + ownership of a given email address. + +:``M_INVALID_ADDRESS``: + The provided third party address was not valid. + +:``M_SEND_ERROR``: + There was an error sending a notification. Typically seen when attempting to + verify ownership of a given third party address. + +:``M_UNRECOGNIZED``: + The request contained an unrecognised value, such as an unknown token or medium. + +:``M_THREEPID_IN_USE``: + The third party identifier is already in use by another user. Typically this + error will have an additional ``mxid`` property to indicate who owns the + third party identifier. + +:``M_UNKNOWN``: + An unknown error has occurred. + Privacy ------- diff --git a/specification/modules/account_data.rst b/specification/modules/account_data.rst index 1c031ee1..d0ed201a 100644 --- a/specification/modules/account_data.rst +++ b/specification/modules/account_data.rst @@ -31,7 +31,7 @@ The client recieves the account data as events in the ``account_data`` sections of a ``/sync``. These events can also be received in a ``/events`` response or in the -``account_data`` section of a room in ``/initialSync``. ``m.tag`` +``account_data`` section of a room in ``/sync``. ``m.tag`` events appearing in ``/events`` will have a ``room_id`` with the room the tags are for. diff --git a/specification/modules/end_to_end_encryption.rst b/specification/modules/end_to_end_encryption.rst index fa461cc2..170b70f9 100644 --- a/specification/modules/end_to_end_encryption.rst +++ b/specification/modules/end_to_end_encryption.rst @@ -283,6 +283,31 @@ Device verification may reach one of several conclusions. For example: decrypted by such a device. For the Olm protocol, this is documented at https://matrix.org/git/olm/about/docs/signing.rst. +Key sharing +----------- + +If Bob has an encrypted conversation with Alice on his computer, and then logs in +through his phone for the first time, he may want to have access to the previously +exchanged messages. To address this issue, events exist for requesting and sending +keys from device to device. + +When a device is missing keys to decrypt messages, it can request the keys by +sending `m.room_key_request`_ to-device messages to other devices with +``action`` set to ``request``. If a device wishes to share the keys with that +device, it can forward the keys to the first device by sending an encrypted +`m.forwarded_room_key`_ to-device message. The first device should then send an +`m.room_key_request`_ to-device message with ``action`` set to +``cancel_request`` to the other devices that it had originally sent the key +request to; a device that receives a ``cancel_request`` should disregard any +previously-received ``request`` message with the same ``request_id`` and +``requesting_device_id``. + +.. NOTE:: + + Key sharing can be a big attack vector, thus it must be done very carefully. + A reasonable stategy is for a user's client to only send keys requested by the + verified devices of the same user. + Messaging Algorithms -------------------- @@ -391,6 +416,12 @@ this check, a client cannot be sure that the sender device owns the private part of the ed25519 key it claims to have in the Olm payload. This is crucial when the ed25519 key corresponds to a verified device. +If a client has multiple sessions established with another device, it should +use the session from which it last received a message. A client may expire old +sessions by defining a maximum number of olm sessions that it will maintain for +each device, and expiring sessions on a Least Recently Used basis. The maximum +number of olm sessions maintained per device should be at least 4. + ``m.megolm.v1.aes-sha2`` ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -464,6 +495,10 @@ Events {{m_room_key_event}} +{{m_room_key_request_event}} + +{{m_forwarded_room_key_event}} + Key management API ~~~~~~~~~~~~~~~~~~ diff --git a/specification/modules/push.rst b/specification/modules/push.rst index 1972fa17..4a6dfb10 100644 --- a/specification/modules/push.rst +++ b/specification/modules/push.rst @@ -744,4 +744,4 @@ should send a "sync" command to instruct the client to get new events from the homeserver directly. -.. _`Push Gateway Specification`: ../push_gateway/unstable.html +.. _`Push Gateway Specification`: ../push_gateway/%PUSH_GATEWAY_RELEASE_LABEL%.html diff --git a/specification/modules/receipts.rst b/specification/modules/receipts.rst index a6d8cbf7..faba7b62 100644 --- a/specification/modules/receipts.rst +++ b/specification/modules/receipts.rst @@ -38,11 +38,10 @@ single ``event_id``. Client behaviour ---------------- -In ``/initialSync``, receipts are listed in a separate top level ``receipts`` -key. In ``/sync``, receipts are contained in the ``ephemeral`` block for a -room. New receipts that come down the event streams are deltas which update -existing mappings. Clients should replace older receipt acknowledgements based -on ``user_id`` and ``receipt_type`` pairs. For example:: +In ``/sync``, receipts are listed under the ``ephemeral`` array of events +for a given room. New receipts that come down the event streams are deltas +which update existing mappings. Clients should replace older receipt acknowledgements +based on ``user_id`` and ``receipt_type`` pairs. For example:: Client receives m.receipt: user = @alice:example.com diff --git a/specification/push_gateway.rst b/specification/push_gateway.rst index e4a9d6ea..e4623887 100644 --- a/specification/push_gateway.rst +++ b/specification/push_gateway.rst @@ -1,4 +1,5 @@ .. Copyright 2016 OpenMarket Ltd +.. Copyright 2018 New Vector Ltd .. .. Licensed under the Apache License, Version 2.0 (the "License"); .. you may not use this file except in compliance with the License. @@ -21,13 +22,27 @@ the homeserver. This is managed by a distinct entity called the Push Gateway. .. contents:: Table of Contents .. sectnum:: -Specification version ---------------------- +Changelog +--------- + +.. topic:: Version: %PUSH_GATEWAY_RELEASE_LABEL% +{{push_gateway_changelog}} This version of the specification is generated from `matrix-doc `_ as of Git commit `{{git_version}} `_. +For the full historical changelog, see +https://github.com/matrix-org/matrix-doc/blob/master/changelogs/push_gateway.rst + +Other versions of this specification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following other versions are also available, in reverse chronological order: + +- `HEAD `_: Includes all changes since the latest versioned release. +- `r0.1.0 `_ + Overview -------- diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 9440f2f2..983da62f 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -71,39 +71,41 @@ This version of the specification is generated from `matrix-doc `_ as of Git commit `{{git_version}} `_. -Server Discovery +Server discovery ---------------- -Resolving Server Names +Resolving server names ~~~~~~~~~~~~~~~~~~~~~~ Each matrix homeserver is identified by a server name consisting of a hostname -and an optional TLS port. +and an optional port, as described by the `grammar +<../appendices.html#server-name>`_. Server names should be resolved to an IP +address and port using the following process: -.. code:: +* If the hostname is an IP literal, then that IP address should be used, + together with the given port number, or 8448 if no port is given. - server_name = hostname [ ":" tls_port] - tls_port = *DIGIT +* Otherwise, if the port is present, then an IP address is discovered by + looking up an AAAA or A record for the hostname, and the specified port is + used. -.. ** +* If the hostname is not an IP literal and no port is given, the server is + discovered by first looking up a ``_matrix._tcp`` SRV record for the + hostname, which may give a hostname (to be looked up using AAAA or A queries) + and port. If the SRV record does not exist, then the server is discovered by + looking up an AAAA or A record on the hostname and taking the default + fallback port number of 8448. -If the port is present then the server is discovered by looking up an AAAA or -A record for the hostname and connecting to the specified TLS port. If the port -is absent then the server is discovered by looking up a ``_matrix._tcp`` SRV -record for the hostname. If this record does not exist then the server is -discovered by looking up an AAAA or A record on the hostname and taking the -default fallback port number of 8448. -Homeservers may use SRV records to load balance requests between multiple TLS -endpoints or to failover to another endpoint if an endpoint fails. + Homeservers may use SRV records to load balance requests between multiple TLS + endpoints or to failover to another endpoint if an endpoint fails. -If the DNS name is a literal IP address, the port specified or the fallback -port should be used. - -When making requests to servers, use the DNS name of the target server in the -``Host`` header, regardless of the host given in the SRV record. For example, -if making a request to ``example.org``, and the SRV record resolves to ``matrix. -example.org``, the ``Host`` header in the request should be ``example.org``. The -port number for target server should not appear in the ``Host`` header. +When making requests to servers, use the hostname of the target server in the +``Host`` header, regardless of any hostname given in the SRV record. For +example, if the server name is ``example.org``, and the SRV record resolves to +``matrix.example.org``, the ``Host`` header in the request should be +``example.org``. If an explicit port was given in the server name, it should be +included in the ``Host`` header; otherwise, no port number should be given in +the ``Host`` header. Server implementation ~~~~~~~~~~~~~~~~~~~~~~ @@ -1109,3 +1111,4 @@ that are too long. .. _`Inviting to a room`: #inviting-to-a-room .. _`Canonical JSON`: ../appendices.html#canonical-json .. _`Unpadded Base64`: ../appendices.html#unpadded-base64 +.. _`Server ACLs`: ../client_server/unstable.html#module-server-acls diff --git a/specification/targets.yaml b/specification/targets.yaml index 446332c9..56e9ec34 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -13,7 +13,7 @@ targets: application_service: files: - application_service_api.rst - version_label: unstable + version_label: "%APPSERVICE_RELEASE_LABEL%" server_server: files: - server_server_api.rst @@ -25,7 +25,7 @@ targets: push_gateway: files: - push_gateway.rst - version_label: unstable + version_label: "%PUSH_GATEWAY_RELEASE_LABEL%" appendices: files: - appendices.rst