From ea307b5bdbae811e8b94e0603c2c0ac5fde9ba0f Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 31 Jul 2018 13:55:53 -0600 Subject: [PATCH 01/10] Support rendering schema definitions in the spec --- .../templating/matrix_templates/sections.py | 14 +++++++ .../templates/schema-definition.tmpl | 21 ++++++++++ scripts/templating/matrix_templates/units.py | 38 +++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 scripts/templating/matrix_templates/templates/schema-definition.tmpl diff --git a/scripts/templating/matrix_templates/sections.py b/scripts/templating/matrix_templates/sections.py index 64e32aa4..1a93c723 100644 --- a/scripts/templating/matrix_templates/sections.py +++ b/scripts/templating/matrix_templates/sections.py @@ -189,3 +189,17 @@ class MatrixSections(Sections): template = self.env.get_template("apis.tmpl") apis = self.units.get("apis") return template.render(apis=apis) + + def render_swagger_definition(self): + rendered = {} + template = self.env.get_template("schema-definition.tmpl") + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] + definitions = self.units.get("swagger_definitions") + for group, swagger_def in definitions.items(): + rendered["definition_" + group] = template.render( + definition=swagger_def['definition'], + examples=swagger_def['examples'], + title_kind=subtitle_title_char) + return rendered \ No newline at end of file diff --git a/scripts/templating/matrix_templates/templates/schema-definition.tmpl b/scripts/templating/matrix_templates/templates/schema-definition.tmpl new file mode 100644 index 00000000..42a7ae47 --- /dev/null +++ b/scripts/templating/matrix_templates/templates/schema-definition.tmpl @@ -0,0 +1,21 @@ +{% import 'tables.tmpl' as tables -%} + +``{{definition.title}}`` Schema +{{(11 + definition.title | length) * title_kind}} + +{% if 'description' in definition %} +{{definition.description}} +{% endif %} + +{% for table in definition.tables -%} +{{"``"+table.title+"``" if table.title else "" }} +{{ tables.paramtable(table.rows) }} +{% endfor %} + +Example{% if examples | length > 1 %}s{% endif %}: + +{% for example in examples %} +.. code:: json + + {{example | jsonify(4, 4)}} +{% endfor %} diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 5b39a2b3..d6579e6c 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -43,6 +43,13 @@ HTTP_APIS = { os.path.join(matrix_doc_dir, "api/push-gateway"): "push", os.path.join(matrix_doc_dir, "api/server-server"): "ss", } +SWAGGER_DEFINITIONS = { + os.path.join(matrix_doc_dir, "api/application-service/definitions"): "as", + os.path.join(matrix_doc_dir, "api/client-server/definitions"): "cs", + #os.path.join(matrix_doc_dir, "api/identity/definitions"): "is", + #os.path.join(matrix_doc_dir, "api/push-gateway/definitions"): "push", + os.path.join(matrix_doc_dir, "api/server-server/definitions"): "ss", +} EVENT_EXAMPLES = os.path.join(matrix_doc_dir, "event-schemas/examples") EVENT_SCHEMA = os.path.join(matrix_doc_dir, "event-schemas/schema") CORE_EVENT_SCHEMA = os.path.join(matrix_doc_dir, "event-schemas/schema/core-event-schema") @@ -654,6 +661,37 @@ class MatrixUnits(Units): apis[group_name] = api return apis + + def load_swagger_definitions(self): + defs = {} + for path, prefix in SWAGGER_DEFINITIONS.items(): + for filename in os.listdir(path): + if not filename.endswith(".yaml"): + continue + filepath = os.path.join(path, filename) + logger.info("Reading swagger definition: %s" % filepath) + with open(filepath, "r") as f: + # strip .yaml + group_name = filename[:-5].replace("-", "_") + group_name = "%s_%s" % (prefix, group_name) + definition = yaml.load(f.read(), OrderedLoader) + definition = resolve_references(filepath, definition) + if 'type' not in definition: + continue + try: + example = get_example_for_schema(definition) + except: + pass # do nothing - we don't care + if 'title' not in definition: + definition['title'] = "NO_TITLE" + definition['tables'] = get_tables_for_schema(definition) + defs[group_name] = { + "definition": definition, + "examples": [example] if example is not None else [], + } + return defs + + def load_common_event_fields(self): """Parse the core event schema files From 5027a9a59a4ca257f6b71f78f6f0fe3811940400 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 31 Jul 2018 13:58:23 -0600 Subject: [PATCH 02/10] Improve documentation around EDUs and PDUs Clarify fields, improve examples, and make the tables in the spec be generated rather than duplicated. --- api/server-server/definitions/edu.yaml | 6 +- .../definitions/unsigned_pdu.yaml | 33 +++++- api/server-server/examples/edu.json | 4 +- api/server-server/examples/unsigned_pdu.json | 3 + specification/server_server_api.rst | 101 +----------------- 5 files changed, 44 insertions(+), 103 deletions(-) diff --git a/api/server-server/definitions/edu.yaml b/api/server-server/definitions/edu.yaml index c89573fe..bc1f0cce 100644 --- a/api/server-server/definitions/edu.yaml +++ b/api/server-server/definitions/edu.yaml @@ -16,7 +16,7 @@ type: object title: Ephemeral Data Unit -description: An ephemeral data unit +description: An ephemeral data unit. example: $ref: "../examples/edu.json" properties: @@ -24,17 +24,15 @@ properties: type: string description: The type of ephemeral message. example: "m.presence" - required: true origin: type: string description: The server name sending the ephemeral message. example: "matrix.org" - required: true destination: type: string description: The server name receiving the ephemeral message. example: "elsewhere.com" - required: true content: type: object description: The content of the ephemeral message. +required: ['edu_type', 'origin', 'destination', 'content'] \ No newline at end of file diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml index 0aeede51..dad01537 100644 --- a/api/server-server/definitions/unsigned_pdu.yaml +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -108,8 +108,39 @@ properties: example: "$def456:matrix.org" unsigned: type: object - description: Additional data added by the origin server but not covered by the ``signatures``. + title: Example Unsigned Data + description: |- + Additional data added by the origin server but not covered by the ``signatures``. More + keys than those defined here may be used. example: {"key": "value"} + properties: + age: + type: integer + description: The number of milliseconds that have passed since this message was sent. + example: 4612 + age_ts: + type: integer + format: int64 + description: The POSIX timestamp this message was sent at in milliseconds. + replaces_state: + type: string + description: The event ID of the state event this event replaces. + example: "$state_event:domain.com" + prev_sender: + type: string + description: The sender of the replaced state event. + example: "@someone:domain.com" + prev_content: + type: string + description: The content of the replaced state event. + example: { + "membership": "join", + "displayname": "Bob" + } + redacted_because: + type: string + description: A reason for why the event was redacted. + example: "Inappropriate content" required: - event_id - room_id diff --git a/api/server-server/examples/edu.json b/api/server-server/examples/edu.json index 95a7b55d..9ff40dc3 100644 --- a/api/server-server/examples/edu.json +++ b/api/server-server/examples/edu.json @@ -1,7 +1,7 @@ { "edu_type": "m.presence", - "origin": "blue", - "destination": "orange", + "origin": "matrix.org", + "destination": "elsewhere.com", "content": { "key": "value" } diff --git a/api/server-server/examples/unsigned_pdu.json b/api/server-server/examples/unsigned_pdu.json index 0b585a63..f4d2e749 100644 --- a/api/server-server/examples/unsigned_pdu.json +++ b/api/server-server/examples/unsigned_pdu.json @@ -20,5 +20,8 @@ ], "content": { "key": "value" + }, + "unsigned": { + "age": 4612 } } \ No newline at end of file diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f1825f27..3ac15fa7 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -284,78 +284,6 @@ PDUs Each PDU contains a single Room Event which the origin server wants to send to the destination. - -PDU Fields -~~~~~~~~~~ - -.. TODO-spec - - Figure out how to embed swagger definitions in here (or improve the section) - -==================== ================== ======================================= - Key Type Description -==================== ================== ======================================= -``room_id`` String **Required**. Room identifier. -``sender`` String **Required**. The ID of the user sending - the event. -``origin`` String **Required**. ``server_name`` of the - homeserver that created this event. -``event_id`` String **Required**. Unique identifier for the - event being sent. -``origin_server_ts`` Integer **Required**. Timestamp in milliseconds - on origin homeserver when this event - was created. -``type`` String **Required**. Event type. -``state_key`` String If this key is present, the event is a - state event, and it will replace - previous events with the same ``type`` - and ``state_key`` in the room state. -``content`` Object **Required**. The content of the event. -``prev_events`` List of (String, **Required**. Event IDs and hashes of - {String: String}) the most recent events in the room that - pairs the homeserver was aware of when it - made this event. -``depth`` Integer **Required**. The maximum depth of the - ``prev_events``, plus one. -``auth_events`` List of (String, **Required**. Event IDs and hashes for - {String: String}) the "auth events" of this event. - pairs -``hashes`` {String: String} **Required**. Hashes of the PDU, - following the algorithm specified in - `Signing Events`_. -``signatures`` {String: **Required**. Signatures of the redacted - {String: String}} PDU, following the algorithm specified - in `Signing Events`_. -``redacts`` String For redaction events, the ID of the - event being redacted. -``unsigned`` Object Additional data added by the origin - server but not covered by the - ``signatures``. -==================== ================== ======================================= - -Example: - -.. code:: json - - { - "room_id": "!UcYsUzyxTGDxLBEvLy:example.org", - "sender": "@alice:example.com", - "origin": "example.com", - "event_id": "$a4ecee13e2accdadf56c1025:example.com", - "origin_server_ts": 1404838188000, - "type": "m.room.message", - "prev_events": [ - ["$af232176:example.org", {"sha256": "abase64encodedsha256hashshouldbe43byteslong"}] - ], - "hashes": {"sha256": "thishashcoversallfieldsincasethisisredacted"}, - "signatures": { - "example.com": { - "ed25519:key_version:": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus" - } - }, - "content": {...} - } - The ``prev_events`` field of a PDU identifies the "parents" of the event, and thus establishes a partial ordering on events within the room by linking them into a Directed Acyclic Graph (DAG). The sending server should populate this @@ -386,6 +314,8 @@ following subset of the room state: - The current ``m.room.join_rules`` event, if any. - The sender's current ``m.room.member`` event, if any. +{{definition_ss_pdu}} + Authorization of PDUs ~~~~~~~~~~~~~~~~~~~~~ @@ -555,32 +485,11 @@ The rules are as follows: EDUs ---- -.. WARNING:: - This section may be misleading or inaccurate. - EDUs, by comparison to PDUs, do not have an ID, a room ID, or a list of -"previous" IDs. The only mandatory fields for these are the type, origin and -destination homeserver names, and the actual nested content. +"previous" IDs. They are intended to be non-persistent data such as user +presence, typing notifications, etc. -======================== ============ ========================================= - Key Type Description -======================== ============ ========================================= -``edu_type`` String The type of the ephemeral message. -``origin`` String The server name sending the ephemeral - message. -``destination`` String The server name receiving the ephemeral - message. -``content`` Object Content of the ephemeral message. -======================== ============ ========================================= - -.. code:: json - - { - "edu_type": "m.presence", - "origin": "blue", - "destination": "orange", - "content": {...} - } +{{definition_ss_edu}} Room State Resolution --------------------- From d0e8df8f3f23df0f2c2c9d60fd9db7029668a7fd Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 31 Jul 2018 14:02:04 -0600 Subject: [PATCH 03/10] prev_content is an object --- api/server-server/definitions/unsigned_pdu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml index dad01537..ab6e5b1c 100644 --- a/api/server-server/definitions/unsigned_pdu.yaml +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -131,7 +131,7 @@ properties: description: The sender of the replaced state event. example: "@someone:domain.com" prev_content: - type: string + type: object description: The content of the replaced state event. example: { "membership": "join", From 4b1955f395b025edcdcd7ff321927a48f2e082d0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 2 Aug 2018 12:06:43 -0600 Subject: [PATCH 04/10] Clarify what the auth_events on a PDU are --- api/server-server/definitions/unsigned_pdu.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml index ab6e5b1c..f042dffb 100644 --- a/api/server-server/definitions/unsigned_pdu.yaml +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -82,7 +82,9 @@ properties: example: 12 auth_events: type: array - description: Event IDs and hashes for the "auth events" of this event. + description: |- + An event reference list containing the authorization events that would + allow this event to be in the room. items: type: array maxItems: 2 From 96f551026695ac0e9a5aa16a960cf80e6fbd72ae Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 3 Aug 2018 12:03:23 -0600 Subject: [PATCH 05/10] Ensure the example is always set to something --- scripts/templating/matrix_templates/units.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index d6579e6c..045826f2 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -681,6 +681,7 @@ class MatrixUnits(Units): try: example = get_example_for_schema(definition) except: + example = None pass # do nothing - we don't care if 'title' not in definition: definition['title'] = "NO_TITLE" From ef9d766dc3551fd23b0410dab9d274a599ab0803 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 3 Aug 2018 14:51:46 -0600 Subject: [PATCH 06/10] Remove irrelevant TODO We fixed the EDU, so we don't need this comment. --- api/server-server/definitions/edu.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/server-server/definitions/edu.yaml b/api/server-server/definitions/edu.yaml index bc1f0cce..725af695 100644 --- a/api/server-server/definitions/edu.yaml +++ b/api/server-server/definitions/edu.yaml @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO: Address any concerns about this being inaccurate (flagged as such in the EDUs section) - type: object title: Ephemeral Data Unit description: An ephemeral data unit. From db6d466fa1db004afc9ea7909224bc794d2e4b51 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 3 Aug 2018 19:49:46 -0600 Subject: [PATCH 07/10] Go one level deep when finding definitions This is useful for when we start defining event schemas. This also has a sanity check for ensuring the directory exists, allowing the IS and push API paths to be uncommented. --- scripts/templating/matrix_templates/units.py | 66 ++++++++++++-------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 045826f2..88f7b86c 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -46,8 +46,8 @@ HTTP_APIS = { SWAGGER_DEFINITIONS = { os.path.join(matrix_doc_dir, "api/application-service/definitions"): "as", os.path.join(matrix_doc_dir, "api/client-server/definitions"): "cs", - #os.path.join(matrix_doc_dir, "api/identity/definitions"): "is", - #os.path.join(matrix_doc_dir, "api/push-gateway/definitions"): "push", + os.path.join(matrix_doc_dir, "api/identity/definitions"): "is", + os.path.join(matrix_doc_dir, "api/push-gateway/definitions"): "push", os.path.join(matrix_doc_dir, "api/server-server/definitions"): "ss", } EVENT_EXAMPLES = os.path.join(matrix_doc_dir, "event-schemas/examples") @@ -665,33 +665,45 @@ class MatrixUnits(Units): def load_swagger_definitions(self): defs = {} for path, prefix in SWAGGER_DEFINITIONS.items(): - for filename in os.listdir(path): - if not filename.endswith(".yaml"): - continue - filepath = os.path.join(path, filename) - logger.info("Reading swagger definition: %s" % filepath) - with open(filepath, "r") as f: - # strip .yaml - group_name = filename[:-5].replace("-", "_") - group_name = "%s_%s" % (prefix, group_name) - definition = yaml.load(f.read(), OrderedLoader) - definition = resolve_references(filepath, definition) - if 'type' not in definition: - continue - try: - example = get_example_for_schema(definition) - except: - example = None - pass # do nothing - we don't care - if 'title' not in definition: - definition['title'] = "NO_TITLE" - definition['tables'] = get_tables_for_schema(definition) - defs[group_name] = { - "definition": definition, - "examples": [example] if example is not None else [], - } + self._load_swagger_definitions_in_dir(defs, path, prefix) return defs + def _load_swagger_definitions_in_dir(self, defs, path, prefix, recurse=True): + if not os.path.exists(path): + return defs + for filename in os.listdir(path): + filepath = os.path.join(path, filename) + if os.path.isdir(filepath) and recurse: + safe_name = re.sub(r"[^a-zA-Z0-9_]", "_", filename) + dir_prefix = "_".join([prefix, safe_name]) + # We don't recurse because we have to stop at some point + self._load_swagger_definitions_in_dir( + defs, filepath, dir_prefix, recurse=False) + if not filename.endswith(".yaml"): + continue + filepath = os.path.join(path, filename) + logger.info("Reading swagger definition: %s" % filepath) + with open(filepath, "r") as f: + # strip .yaml + group_name = re.sub(r"[^a-zA-Z0-9_]", "_", filename[:-5]) + group_name = "%s_%s" % (prefix, group_name) + definition = yaml.load(f.read(), OrderedLoader) + definition = resolve_references(filepath, definition) + if 'type' not in definition: + continue + try: + example = get_example_for_schema(definition) + except: + example = None + pass # do nothing - we don't care + if 'title' not in definition: + definition['title'] = "NO_TITLE" + definition['tables'] = get_tables_for_schema(definition) + defs[group_name] = { + "definition": definition, + "examples": [example] if example is not None else [], + } + return defs def load_common_event_fields(self): """Parse the core event schema files From 7ada91787a9a6be86632566911d29d7145fbda5c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 7 Aug 2018 14:20:46 -0600 Subject: [PATCH 08/10] age_ts isn't real --- api/server-server/definitions/unsigned_pdu.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml index f042dffb..ab281224 100644 --- a/api/server-server/definitions/unsigned_pdu.yaml +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -120,10 +120,6 @@ properties: type: integer description: The number of milliseconds that have passed since this message was sent. example: 4612 - age_ts: - type: integer - format: int64 - description: The POSIX timestamp this message was sent at in milliseconds. replaces_state: type: string description: The event ID of the state event this event replaces. From cfdbee5fc27a4da2c0d1cb96ffd94e31bc561e91 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 7 Aug 2018 14:21:01 -0600 Subject: [PATCH 09/10] EDU origin and destinations aren't required --- api/server-server/definitions/edu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server-server/definitions/edu.yaml b/api/server-server/definitions/edu.yaml index 725af695..142f4b93 100644 --- a/api/server-server/definitions/edu.yaml +++ b/api/server-server/definitions/edu.yaml @@ -33,4 +33,4 @@ properties: content: type: object description: The content of the ephemeral message. -required: ['edu_type', 'origin', 'destination', 'content'] \ No newline at end of file +required: ['edu_type', 'content'] \ No newline at end of file From 57eeddd849e505e19f046f0bff52a57d916330b7 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 8 Aug 2018 07:43:22 -0600 Subject: [PATCH 10/10] Remove unused origin and destination on EDUs --- api/server-server/definitions/edu.yaml | 8 -------- api/server-server/examples/edu.json | 2 -- 2 files changed, 10 deletions(-) diff --git a/api/server-server/definitions/edu.yaml b/api/server-server/definitions/edu.yaml index 142f4b93..0e4edcc6 100644 --- a/api/server-server/definitions/edu.yaml +++ b/api/server-server/definitions/edu.yaml @@ -22,14 +22,6 @@ properties: type: string description: The type of ephemeral message. example: "m.presence" - origin: - type: string - description: The server name sending the ephemeral message. - example: "matrix.org" - destination: - type: string - description: The server name receiving the ephemeral message. - example: "elsewhere.com" content: type: object description: The content of the ephemeral message. diff --git a/api/server-server/examples/edu.json b/api/server-server/examples/edu.json index 9ff40dc3..f5a58e21 100644 --- a/api/server-server/examples/edu.json +++ b/api/server-server/examples/edu.json @@ -1,7 +1,5 @@ { "edu_type": "m.presence", - "origin": "matrix.org", - "destination": "elsewhere.com", "content": { "key": "value" }