From 7448ef0db0166450d23b302c589202f1376034f2 Mon Sep 17 00:00:00 2001 From: Konstantinos Sideris Date: Tue, 3 Apr 2018 13:18:29 +0300 Subject: [PATCH 001/308] Use the correct event_id format on examples Signed-off-by: Konstantinos Sideris --- api/client-server/redaction.yaml | 2 +- api/client-server/room_send.yaml | 2 +- api/client-server/room_state.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/client-server/redaction.yaml b/api/client-server/redaction.yaml index 752d841a..58141049 100644 --- a/api/client-server/redaction.yaml +++ b/api/client-server/redaction.yaml @@ -79,7 +79,7 @@ paths: description: "An ID for the redaction event." examples: application/json: { - "event_id": "YUwQidLecu" + "event_id": "$YUwQidLecu:example.com" } schema: type: object diff --git a/api/client-server/room_send.yaml b/api/client-server/room_send.yaml index 0bb1bbcb..6963f76c 100644 --- a/api/client-server/room_send.yaml +++ b/api/client-server/room_send.yaml @@ -76,7 +76,7 @@ paths: description: "An ID for the sent event." examples: application/json: { - "event_id": "YUwRidLecu" + "event_id": "$YUwRidLecu:example.com" } schema: type: object diff --git a/api/client-server/room_state.yaml b/api/client-server/room_state.yaml index 042b5eed..c04fb803 100644 --- a/api/client-server/room_state.yaml +++ b/api/client-server/room_state.yaml @@ -78,7 +78,7 @@ paths: description: "An ID for the sent event." examples: application/json: { - "event_id": "YUwRidLecu" + "event_id": "$YUwRidLecu:example.com" } schema: type: object @@ -133,7 +133,7 @@ paths: description: "An ID for the sent event." examples: application/json: { - "event_id": "YUwRidLecu" + "event_id": "$YUwRidLecu:example.com" } schema: type: object From e18aa30fd26cf77ad27825381371ad924adddc9d Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Wed, 2 May 2018 13:47:01 +0100 Subject: [PATCH 002/308] start generating new proposals page --- scripts/proposals.py | 50 +++++++++++++++++++++++++++++++ specification/proposals.rst | 5 ++++ specification/proposals_intro.rst | 20 +++++++++++++ specification/targets.yaml | 4 +++ 4 files changed, 79 insertions(+) create mode 100644 scripts/proposals.py create mode 100644 specification/proposals.rst create mode 100644 specification/proposals_intro.rst diff --git a/scripts/proposals.py b/scripts/proposals.py new file mode 100644 index 00000000..1e4b8683 --- /dev/null +++ b/scripts/proposals.py @@ -0,0 +1,50 @@ +# proposals.py: generate an RST file (proposals.rst) from queries to github.com/matrix.org/matrix-doc/issues. +# v0.0.1 +# todo: +# use status labels to create separate sections. +# include all fields discussed at https://docs.google.com/document/d/1wLln7da12l0H5YgAh5xM2TVE7VsTjXzhEwVh3sRBMCk/edit# +# format, probably as tables + +import requests +import re +from m2r import convert as m2r + +def getpage(url, page): + resp = requests.get(url + str(page)) + json.extend(resp.json()) + + for link in resp.links.values(): + if link['rel'] == 'last': + return re.search('page=(.+?)', link['url']).group(1) + +json = list() +print("json:" + str(len(json))) +pagecount = getpage('https://api.github.com/repos/matrix-org/matrix-doc/issues?labels=spec-omission&state=open&page=', 1) +print("json:" + str(len(json))) +print("pagecount:" + str(pagecount)) +for page in range(2, int(pagecount) + 1): + getpage('https://api.github.com/repos/matrix-org/matrix-doc/issues?labels=spec-omission&state=open&page=', page) + print("json:" + str(len(json))) + print("currentpage:" + str(page)) + print("pagecount:" + str(pagecount)) + print("json:" + str(len(json))) + + +text_file = open("../specification/proposals.rst", "w") + +text_file.write(".. contents:: Table of Contents\n") +text_file.write(".. sectnum::") +text_file.write("\n") +text_file.write("\n") +text_file.write("The Proposals List\n------------------\n") +# text_file.write(json[0]['user']['login']) + +for item in json: + prop_header = item['title'] + " (" + str(item['number']) + ")" + text_file.write(prop_header + "\n") + text_file.write("~" * len(prop_header)) + text_file.write("\n\n") + body = m2r(str(item['body'])) + text_file.write(body + "\n\n\n") + +text_file.close() diff --git a/specification/proposals.rst b/specification/proposals.rst new file mode 100644 index 00000000..5a1f02c0 --- /dev/null +++ b/specification/proposals.rst @@ -0,0 +1,5 @@ +.. contents:: Table of Contents +.. sectnum:: + +The Proposals List +------------------ diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst new file mode 100644 index 00000000..672c3468 --- /dev/null +++ b/specification/proposals_intro.rst @@ -0,0 +1,20 @@ +.. contents:: Table of Contents +.. sectnum:: + +Proposals for Spec Changes to Matrix +------------------------------------ + +Proposal Tracking +----------------- + +This is a living document generated from the list of proposals at `matrix-doc/issues `_ on GitHub. + +We use labels and some metadata in the issue text to generate this page. When adding or updating an issue, make sure you specify the current status as a label: + +- WIP +- Ready for review +- In review +- Merged +- Rejected +- Stalled + diff --git a/specification/targets.yaml b/specification/targets.yaml index 50a9fb8d..8ca5ce30 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -38,6 +38,10 @@ targets: - appendices/threepids.rst - appendices/threat_model.rst - appendices/test_vectors.rst + proposals: + files: + - proposals_intro.rst + - proposals.rst groups: # reusable blobs of files when prefixed with 'group:' modules: - modules/instant_messaging.rst From 2783820f98b02f01dd2d55aca1e05fc3e78a617d Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 3 May 2018 12:23:24 +0100 Subject: [PATCH 003/308] Initial draft of GET media/limits --- api/client-server/content-repo.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 03a25fd9..6e3ba273 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -269,3 +269,30 @@ paths: "$ref": "definitions/error.yaml" tags: - Media + "/limits": + get: + summary: Get limits on what can be uploaded to the content repository. + Clients SHOULD use this as a guide when uploading content. + All values are intentionally left optional, the client MUST assume + that any field not given is not limited. + + **NOTE:** Reverse proxies may apply their own limits. + operationId: getLimits + produces: ["application/json"] + security: + - accessToken: [] + responses: + 200: + description: The limits exposed by the matrix server. + schema: + type: object + properties: + size: + type: number + description: "The maximum size a upload can be in bytes." + examples: + application/json: { + "size": 50000000 + } + tags: + - Media From c9abf36097f1f4a3a2086c6629e9ab8112c035d2 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 3 May 2018 16:58:22 +0100 Subject: [PATCH 004/308] size > upload_size --- api/client-server/content-repo.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 6e3ba273..cfcb00dd 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -287,12 +287,12 @@ paths: schema: type: object properties: - size: + upload_size: type: number description: "The maximum size a upload can be in bytes." examples: application/json: { - "size": 50000000 + "upload_size": 50000000 } tags: - Media From de6fe1b76e325380eef7110364c0e11ac2d014d2 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 3 May 2018 18:00:50 +0100 Subject: [PATCH 005/308] Add note about auth --- api/client-server/content-repo.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index cfcb00dd..97a7d8f9 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -277,6 +277,9 @@ paths: that any field not given is not limited. **NOTE:** Reverse proxies may apply their own limits. + + If auth is not supplied, this endpoint gives the global limit of the server. + Otherwise it should give the limits applied to the authenticated user. operationId: getLimits produces: ["application/json"] security: From ffc8ee298adfc90a2b21801636927cbcff78b969 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Tue, 8 May 2018 11:50:34 +0100 Subject: [PATCH 006/308] a upload > an upload --- api/client-server/content-repo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 97a7d8f9..cf7b5f25 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -292,7 +292,7 @@ paths: properties: upload_size: type: number - description: "The maximum size a upload can be in bytes." + description: "The maximum size an upload can be in bytes." examples: application/json: { "upload_size": 50000000 From 1f7fbefd53394120411d8af10d0198548cfe8d27 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Tue, 8 May 2018 12:17:38 +0100 Subject: [PATCH 007/308] start using a table --- scripts/proposals.py | 50 +++++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index 1e4b8683..4ba4d0ee 100644 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -7,6 +7,7 @@ import requests import re +from datetime import datetime from m2r import convert as m2r def getpage(url, page): @@ -18,32 +19,57 @@ def getpage(url, page): return re.search('page=(.+?)', link['url']).group(1) json = list() -print("json:" + str(len(json))) +# labels = ['p1', p2] +# new status labels: +# proposal-ready-for-review,rejected,stalled,merged,spec-pr-in-review,proposal-wip,proposal-in-review,spec-pr-ready-for-review,proposal-passed-review +urlbase = 'https://api.github.com/repos/matrix-org/matrix-doc/issues?labels=p1,p2,p3,p4,p5&state=open&page=' pagecount = getpage('https://api.github.com/repos/matrix-org/matrix-doc/issues?labels=spec-omission&state=open&page=', 1) -print("json:" + str(len(json))) -print("pagecount:" + str(pagecount)) for page in range(2, int(pagecount) + 1): - getpage('https://api.github.com/repos/matrix-org/matrix-doc/issues?labels=spec-omission&state=open&page=', page) - print("json:" + str(len(json))) - print("currentpage:" + str(page)) - print("pagecount:" + str(pagecount)) - print("json:" + str(len(json))) + getpage(urlbase, page) text_file = open("../specification/proposals.rst", "w") -text_file.write(".. contents:: Table of Contents\n") -text_file.write(".. sectnum::") -text_file.write("\n") +text_file.write("Tables\n------------------\n\n.. list-table::\n :header-rows: 1\n\n") + +text_file.write(" * - ID\n") +text_file.write(" - github username\n") +text_file.write(" - proposal title\n") +text_file.write(" - created_at\n") +text_file.write(" - updated_at\n") +text_file.write(" - maindoc\n") + +#`matrix-doc/issues `_ +for item in json: + maindoc = re.search('Documentation: (.+?)\n', str(item['body'])) + if maindoc is not None: + maindoc = maindoc.group(1) + text_file.write(" * - `" + str(item['number']) + " <" + item['html_url'] + ">`_\n") + text_file.write(" - " + item['user']['login'] + "\n") + text_file.write(" - " + item['title'] + "\n") + text_file.write(" - " + item['created_at'] + "\n") + text_file.write(" - " + item['updated_at'] + "\n") + text_file.write(" - " + str(maindoc) + "\n") + text_file.write("\n") + + text_file.write("The Proposals List\n------------------\n") # text_file.write(json[0]['user']['login']) - for item in json: + # write a header prop_header = item['title'] + " (" + str(item['number']) + ")" text_file.write(prop_header + "\n") text_file.write("~" * len(prop_header)) text_file.write("\n\n") + + # write some metadata + text_file.write(item['created_at'] + "\n") + text_file.write(item['updated_at'] + "\n") + # created = datetime.strptime(item['created_at'], "%Y-%m-%dT%XZ") + + + # write body text body = m2r(str(item['body'])) text_file.write(body + "\n\n\n") From c58dc59ffe89c3f3e8bf232a53ea490ef7861a55 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Tue, 8 May 2018 14:43:06 +0100 Subject: [PATCH 008/308] separate tables by label --- scripts/proposals.py | 99 ++++++++++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 41 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index 4ba4d0ee..171db069 100644 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -10,67 +10,84 @@ import re from datetime import datetime from m2r import convert as m2r +pagecount = 1 + def getpage(url, page): resp = requests.get(url + str(page)) - json.extend(resp.json()) for link in resp.links.values(): if link['rel'] == 'last': - return re.search('page=(.+?)', link['url']).group(1) + pagecount = re.search('page=(.+?)', link['url']).group(1) + + return resp.json() + +def getbylabel(label): + pagecount = 1 + json = list() + urlbase = 'https://api.github.com/repos/matrix-org/matrix-doc/issues?state=open&labels=' + label + '&page=' + print(urlbase) + json.extend(getpage(urlbase, 1)) + for page in range(2, int(pagecount) + 1): + getpage(urlbase, page) + + return json -json = list() -# labels = ['p1', p2] # new status labels: # proposal-ready-for-review,rejected,stalled,merged,spec-pr-in-review,proposal-wip,proposal-in-review,spec-pr-ready-for-review,proposal-passed-review -urlbase = 'https://api.github.com/repos/matrix-org/matrix-doc/issues?labels=p1,p2,p3,p4,p5&state=open&page=' -pagecount = getpage('https://api.github.com/repos/matrix-org/matrix-doc/issues?labels=spec-omission&state=open&page=', 1) -for page in range(2, int(pagecount) + 1): - getpage(urlbase, page) +labels = ['p1', 'p2'] +issues = {} +for label in labels: + issues[label] = getbylabel(label) + print(issues) text_file = open("../specification/proposals.rst", "w") -text_file.write("Tables\n------------------\n\n.. list-table::\n :header-rows: 1\n\n") +text_file.write("Tables\n------------------\n\n") -text_file.write(" * - ID\n") -text_file.write(" - github username\n") -text_file.write(" - proposal title\n") -text_file.write(" - created_at\n") -text_file.write(" - updated_at\n") -text_file.write(" - maindoc\n") -#`matrix-doc/issues `_ -for item in json: - maindoc = re.search('Documentation: (.+?)\n', str(item['body'])) - if maindoc is not None: - maindoc = maindoc.group(1) - text_file.write(" * - `" + str(item['number']) + " <" + item['html_url'] + ">`_\n") - text_file.write(" - " + item['user']['login'] + "\n") - text_file.write(" - " + item['title'] + "\n") - text_file.write(" - " + item['created_at'] + "\n") - text_file.write(" - " + item['updated_at'] + "\n") - text_file.write(" - " + str(maindoc) + "\n") +for label in labels: + text_file.write(label + "\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n") + text_file.write(".. list-table::\n :header-rows: 1\n :widths: auto\n\n") + text_file.write(" * - ID\n") + text_file.write(" - github username\n") + text_file.write(" - proposal title\n") + text_file.write(" - created_at\n") + text_file.write(" - updated_at\n") + text_file.write(" - maindoc\n") + + for item in issues[label]: + maindoc = re.search('Documentation: (.+?)\n', str(item['body'])) + if maindoc is not None: + maindoc = maindoc.group(1) + text_file.write(" * - `" + str(item['number']) + " <" + item['html_url'] + ">`_\n") + text_file.write(" - " + item['user']['login'] + "\n") + text_file.write(" - " + item['title'] + "\n") + text_file.write(" - " + item['created_at'] + "\n") + text_file.write(" - " + item['updated_at'] + "\n") + text_file.write(" - " + str(maindoc) + "\n") + text_file.write("\n\n\n") text_file.write("\n") -text_file.write("The Proposals List\n------------------\n") -# text_file.write(json[0]['user']['login']) -for item in json: - # write a header - prop_header = item['title'] + " (" + str(item['number']) + ")" - text_file.write(prop_header + "\n") - text_file.write("~" * len(prop_header)) - text_file.write("\n\n") +# text_file.write("The Proposals List\n------------------\n") +# # text_file.write(json[0]['user']['login']) +# for item in json: +# # write a header +# prop_header = item['title'] + " (" + str(item['number']) + ")" +# text_file.write(prop_header + "\n") +# text_file.write("~" * len(prop_header)) +# text_file.write("\n\n") - # write some metadata - text_file.write(item['created_at'] + "\n") - text_file.write(item['updated_at'] + "\n") - # created = datetime.strptime(item['created_at'], "%Y-%m-%dT%XZ") +# # write some metadata +# text_file.write(item['created_at'] + "\n") +# text_file.write(item['updated_at'] + "\n") +# # created = datetime.strptime(item['created_at'], "%Y-%m-%dT%XZ") - # write body text - body = m2r(str(item['body'])) - text_file.write(body + "\n\n\n") +# # write body text +# body = m2r(str(item['body'])) +# text_file.write(body + "\n\n\n") text_file.close() From 9e37d15c8fce944c458068f7c94aae7c08daed02 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Tue, 8 May 2018 15:11:15 +0100 Subject: [PATCH 009/308] little ascii diagram of process --- specification/proposals_intro.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index 672c3468..ebc43b9a 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -4,6 +4,7 @@ Proposals for Spec Changes to Matrix ------------------------------------ + Proposal Tracking ----------------- @@ -18,3 +19,28 @@ We use labels and some metadata in the issue text to generate this page. When ad - Rejected - Stalled +:: + + +----------+ +-------------------+ +-----------+ +---------------+ + | | | | | | | | + | Proposal +-----> Proposal +-----> Proposal +-----> Proposal +------+ + | WIP | | Ready for Review | | In Review | | Passed Review | | + | | | | | | | | | + +----------+ +-------------------+ +-----------+ +---------------+ | + | + +---------------------------------------------------------------------------------------+ + | + | +---------------------+ +-----------+ +-----------+ + | | | | | |-----------| + +--> Spec +-----> Spec PR | || || + | PR Ready for Review | | In Review +-----------------------> Merged || + | | | | || || + +---------------------+ +-+-------+-+ |-----------| + | | +-----------+ + +-------+ +--------+ + | | + +-----v----+ +-----v----+ + | | | | + | Stalled | | Rejected | + | | | | + +----------+ +----------+ From a78f7b38e9d565d7e1af4d62159a4296bf65ede2 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Tue, 8 May 2018 15:35:46 +0100 Subject: [PATCH 010/308] generator is now usable --- scripts/proposals.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index 171db069..5bc3b958 100644 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -33,8 +33,10 @@ def getbylabel(label): return json # new status labels: -# proposal-ready-for-review,rejected,stalled,merged,spec-pr-in-review,proposal-wip,proposal-in-review,spec-pr-ready-for-review,proposal-passed-review -labels = ['p1', 'p2'] +# labels = ['proposal-wip', 'proposal-ready-for-review', +# 'proposal-in-review', 'proposal-passed-review', +# 'spec-pr-ready-for-review', 'spec-pr-in-review', 'merged', 'stalled', 'rejected' ] +labels = ['p1', 'p2', 'p3', 'p4', 'p5'] issues = {} for label in labels: @@ -49,23 +51,33 @@ text_file.write("Tables\n------------------\n\n") for label in labels: text_file.write(label + "\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n") text_file.write(".. list-table::\n :header-rows: 1\n :widths: auto\n\n") - text_file.write(" * - ID\n") + text_file.write(" * - MSC\n") text_file.write(" - github username\n") text_file.write(" - proposal title\n") text_file.write(" - created_at\n") text_file.write(" - updated_at\n") text_file.write(" - maindoc\n") + text_file.write(" - author\n") + text_file.write(" - shepherd\n") for item in issues[label]: - maindoc = re.search('Documentation: (.+?)\n', str(item['body'])) - if maindoc is not None: - maindoc = maindoc.group(1) - text_file.write(" * - `" + str(item['number']) + " <" + item['html_url'] + ">`_\n") + text_file.write(" * - `MSC" + str(item['number']) + " <" + item['html_url'] + ">`_\n") text_file.write(" - " + item['user']['login'] + "\n") text_file.write(" - " + item['title'] + "\n") - text_file.write(" - " + item['created_at'] + "\n") - text_file.write(" - " + item['updated_at'] + "\n") + created = datetime.strptime(item['created_at'], "%Y-%m-%dT%XZ") + text_file.write(" - " + created.strftime('%Y-%m-%d') + "\n") + updated = datetime.strptime(item['updated_at'], "%Y-%m-%dT%XZ") + text_file.write(" - " + updated.strftime('%Y-%m-%d') + "\n") + maindoc = re.search('Documentation: (.+?)\n', str(item['body'])) + if maindoc is not None: maindoc = maindoc.group(1) text_file.write(" - " + str(maindoc) + "\n") + author = re.search('Author: (.+?)\n', str(item['body'])) + if author is not None: author = author.group(1) + #if author is None: author = item['user']['login'] + text_file.write(" - " + str(author) + "\n") + shepherd = re.search('Shepherd: (.+?)\n', str(item['body'])) + if shepherd is not None: shepherd = shepherd.group(1) + text_file.write(" - " + str(shepherd) + "\n") text_file.write("\n\n\n") text_file.write("\n") From dc2b53318b38f84d0a7e373a5450452c779ddd8b Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Wed, 9 May 2018 11:32:31 +0100 Subject: [PATCH 011/308] prettify ID column --- scripts/proposals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index 5bc3b958..62a94dba 100644 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -50,7 +50,7 @@ text_file.write("Tables\n------------------\n\n") for label in labels: text_file.write(label + "\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n") - text_file.write(".. list-table::\n :header-rows: 1\n :widths: auto\n\n") + text_file.write(".. list-table::\n :header-rows: 1\n :widths: auto\n :stub-columns: 1\n\n") text_file.write(" * - MSC\n") text_file.write(" - github username\n") text_file.write(" - proposal title\n") From 866b0b6348dc6a052f41c76319e8b62b61cf915a Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Thu, 10 May 2018 16:02:55 +0100 Subject: [PATCH 012/308] I mostly play with ascii art --- scripts/proposals.py | 15 +++++--- specification/proposals_intro.rst | 64 ++++++++++++++++++++----------- 2 files changed, 50 insertions(+), 29 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index 62a94dba..075e3118 100644 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -33,10 +33,10 @@ def getbylabel(label): return json # new status labels: -# labels = ['proposal-wip', 'proposal-ready-for-review', -# 'proposal-in-review', 'proposal-passed-review', -# 'spec-pr-ready-for-review', 'spec-pr-in-review', 'merged', 'stalled', 'rejected' ] -labels = ['p1', 'p2', 'p3', 'p4', 'p5'] +labels = ['proposal-wip', 'proposal-ready-for-review', + 'proposal-in-review', 'proposal-passed-review', + 'spec-pr-ready-for-review', 'spec-pr-in-review', 'merged', 'abandoned', 'rejected', 'blocked' ] +#labels = ['p1', 'p2', 'p3', 'p4', 'p5'] issues = {} for label in labels: @@ -49,6 +49,9 @@ text_file.write("Tables\n------------------\n\n") for label in labels: + if (len(issues[label]) == 0): + continue + text_file.write(label + "\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n") text_file.write(".. list-table::\n :header-rows: 1\n :widths: auto\n :stub-columns: 1\n\n") text_file.write(" * - MSC\n") @@ -68,10 +71,10 @@ for label in labels: text_file.write(" - " + created.strftime('%Y-%m-%d') + "\n") updated = datetime.strptime(item['updated_at'], "%Y-%m-%dT%XZ") text_file.write(" - " + updated.strftime('%Y-%m-%d') + "\n") - maindoc = re.search('Documentation: (.+?)\n', str(item['body'])) + maindoc = re.search('^Documentation: (.+?)\n', str(item['body'])) if maindoc is not None: maindoc = maindoc.group(1) text_file.write(" - " + str(maindoc) + "\n") - author = re.search('Author: (.+?)\n', str(item['body'])) + author = re.search('^Author: (.+?)\n', str(item['body']), flags=re.MULTILINE) if author is not None: author = author.group(1) #if author is None: author = item['user']['login'] text_file.write(" - " + str(author) + "\n") diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index ebc43b9a..3f3f5caa 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -21,26 +21,44 @@ We use labels and some metadata in the issue text to generate this page. When ad :: - +----------+ +-------------------+ +-----------+ +---------------+ - | | | | | | | | - | Proposal +-----> Proposal +-----> Proposal +-----> Proposal +------+ - | WIP | | Ready for Review | | In Review | | Passed Review | | - | | | | | | | | | - +----------+ +-------------------+ +-----------+ +---------------+ | - | - +---------------------------------------------------------------------------------------+ - | - | +---------------------+ +-----------+ +-----------+ - | | | | | |-----------| - +--> Spec +-----> Spec PR | || || - | PR Ready for Review | | In Review +-----------------------> Merged || - | | | | || || - +---------------------+ +-+-------+-+ |-----------| - | | +-----------+ - +-------+ +--------+ - | | - +-----v----+ +-----v----+ - | | | | - | Stalled | | Rejected | - | | | | - +----------+ +----------+ + + + + Proposals | Spec PRs | Other States + +-------+ | +------+ | +----------+ + | | + | | + +----------+ | +------------------+ | +---------+ + | | | | | | | | + | Proposal | | +-> Spec PR | | | Blocked | + | WIP | | | | Ready for Review | | | | + | | | | | | | +---------+ + +----+-----+ | | +---------+--------+ | + | | | | | + | | | | | +-----------+ + +--------v----------+ | | +-----v-----+ | | | + | | | | | | | | Abandoned | + | Proposal | | | | Spec PR | | | | + | Ready for Review | | | | In Review | | +-----------+ + | | | | | | | + +----------+--------+ | | +-----+-----+ | +-----------+ + | | | | | | | + | | | | | | Obsolete | + +------v----+ | | | | | | + | | | | +-----------+ | +-----------+ + | Proposal | | | |-----v-----| | + | In Review | | | || || | + | | | | || Merged || | +----------+ + +----+------+ | | || || | | | + | | | |-----------| | | Rejected | + | | | +-----------+ | | | + +------v--------+ | | | +----------+ + | | | | | + | Proposal | | | | + | Passed Review | | | | + | | | | | + +-------+-------+ | | | + | | | | + | | | | + +---------------+ | + | | + + + + From 73b7faddd4565cfbe9a6c17f0ddf263174feb73c Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Fri, 11 May 2018 20:47:27 +0100 Subject: [PATCH 013/308] handle multiple Google Docs --- scripts/proposals.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index 075e3118..dcd49b88 100644 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -51,7 +51,7 @@ text_file.write("Tables\n------------------\n\n") for label in labels: if (len(issues[label]) == 0): continue - + text_file.write(label + "\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n") text_file.write(".. list-table::\n :header-rows: 1\n :widths: auto\n :stub-columns: 1\n\n") text_file.write(" * - MSC\n") @@ -72,8 +72,13 @@ for label in labels: updated = datetime.strptime(item['updated_at'], "%Y-%m-%dT%XZ") text_file.write(" - " + updated.strftime('%Y-%m-%d') + "\n") maindoc = re.search('^Documentation: (.+?)\n', str(item['body'])) - if maindoc is not None: maindoc = maindoc.group(1) - text_file.write(" - " + str(maindoc) + "\n") + if maindoc is not None: + maindoc = maindoc.group(1) + doc_list_formatted = ["`" + str(item['number']) + "-" + str(i) + " <" + x.strip() + ">`_" for i, x in enumerate(maindoc.split(','),1)] + text_file.write(" - " + ', '.join(doc_list_formatted)) + else: + text_file.write(" - ") + text_file.write("\n") author = re.search('^Author: (.+?)\n', str(item['body']), flags=re.MULTILINE) if author is not None: author = author.group(1) #if author is None: author = item['user']['login'] From cbdd33a5f5da544295fe4ff130678f59337555ea Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Fri, 11 May 2018 21:23:06 +0100 Subject: [PATCH 014/308] single authors and shepherds link correctly --- scripts/proposals.py | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index dcd49b88..138a589c 100644 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -11,6 +11,7 @@ from datetime import datetime from m2r import convert as m2r pagecount = 1 +authors = set() def getpage(url, page): resp = requests.get(url + str(page)) @@ -55,7 +56,7 @@ for label in labels: text_file.write(label + "\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n") text_file.write(".. list-table::\n :header-rows: 1\n :widths: auto\n :stub-columns: 1\n\n") text_file.write(" * - MSC\n") - text_file.write(" - github username\n") + #text_file.write(" - github username\n") text_file.write(" - proposal title\n") text_file.write(" - created_at\n") text_file.write(" - updated_at\n") @@ -65,7 +66,7 @@ for label in labels: for item in issues[label]: text_file.write(" * - `MSC" + str(item['number']) + " <" + item['html_url'] + ">`_\n") - text_file.write(" - " + item['user']['login'] + "\n") + #text_file.write(" - " + item['user']['login'] + "\n") text_file.write(" - " + item['title'] + "\n") created = datetime.strptime(item['created_at'], "%Y-%m-%dT%XZ") text_file.write(" - " + created.strftime('%Y-%m-%d') + "\n") @@ -81,33 +82,19 @@ for label in labels: text_file.write("\n") author = re.search('^Author: (.+?)\n', str(item['body']), flags=re.MULTILINE) if author is not None: author = author.group(1) - #if author is None: author = item['user']['login'] - text_file.write(" - " + str(author) + "\n") + else: author = "@" + item['user']['login'] + authors.add(author.strip()) + text_file.write(" - `" + str(author.strip()) + "`_" + "\n") shepherd = re.search('Shepherd: (.+?)\n', str(item['body'])) - if shepherd is not None: shepherd = shepherd.group(1) + if shepherd is not None: + authors.add(shepherd.group(1).strip()) + shepherd = "`" + shepherd.group(1).strip() + "`_" text_file.write(" - " + str(shepherd) + "\n") text_file.write("\n\n\n") text_file.write("\n") - -# text_file.write("The Proposals List\n------------------\n") -# # text_file.write(json[0]['user']['login']) -# for item in json: -# # write a header -# prop_header = item['title'] + " (" + str(item['number']) + ")" -# text_file.write(prop_header + "\n") -# text_file.write("~" * len(prop_header)) -# text_file.write("\n\n") - -# # write some metadata -# text_file.write(item['created_at'] + "\n") -# text_file.write(item['updated_at'] + "\n") -# # created = datetime.strptime(item['created_at'], "%Y-%m-%dT%XZ") - - -# # write body text -# body = m2r(str(item['body'])) -# text_file.write(body + "\n\n\n") +for author in authors: + text_file.write("\n.. _" + author + ": https://github.com/" + author[1:]) text_file.close() From 3b84de383c2b825f3197edc6ddfe4f01b23f90e2 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Sat, 12 May 2018 10:30:52 +0100 Subject: [PATCH 015/308] cleanup and include all content --- scripts/proposals.py | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index 138a589c..c3d65678 100644 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -56,7 +56,6 @@ for label in labels: text_file.write(label + "\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n") text_file.write(".. list-table::\n :header-rows: 1\n :widths: auto\n :stub-columns: 1\n\n") text_file.write(" * - MSC\n") - #text_file.write(" - github username\n") text_file.write(" - proposal title\n") text_file.write(" - created_at\n") text_file.write(" - updated_at\n") @@ -65,13 +64,26 @@ for label in labels: text_file.write(" - shepherd\n") for item in issues[label]: + # MSC number text_file.write(" * - `MSC" + str(item['number']) + " <" + item['html_url'] + ">`_\n") - #text_file.write(" - " + item['user']['login'] + "\n") + + # title from Github issue text_file.write(" - " + item['title'] + "\n") - created = datetime.strptime(item['created_at'], "%Y-%m-%dT%XZ") - text_file.write(" - " + created.strftime('%Y-%m-%d') + "\n") + + # created date, find local field, otherwise Github + created = re.search('^Date: (.+?)\n', str(item['body']), flags=re.MULTILINE) + if created is not None: + created = created.group(1).strip() + else : + created = datetime.strptime(item['created_at'], "%Y-%m-%dT%XZ") + created = created.strftime('%Y-%m-%d') + text_file.write(" - " + created + "\n") + + # last updated, purely Github updated = datetime.strptime(item['updated_at'], "%Y-%m-%dT%XZ") text_file.write(" - " + updated.strftime('%Y-%m-%d') + "\n") + + # list of document links (urls comma-separated) maindoc = re.search('^Documentation: (.+?)\n', str(item['body'])) if maindoc is not None: maindoc = maindoc.group(1) @@ -80,11 +92,23 @@ for label in labels: else: text_file.write(" - ") text_file.write("\n") + + # author list, if missing just use Github issue creator author = re.search('^Author: (.+?)\n', str(item['body']), flags=re.MULTILINE) - if author is not None: author = author.group(1) - else: author = "@" + item['user']['login'] - authors.add(author.strip()) - text_file.write(" - `" + str(author.strip()) + "`_" + "\n") + if author is not None: + author_list_formatted = set() + author_list = author.group(1) + for a in author_list.split(","): + authors.add(a.strip()) + author_list_formatted.add("`" + str(a.strip()) + "`_") + text_file.write(" - " + ', '.join(author_list_formatted)) + else: + author = "@" + item['user']['login'] + authors.add(author) + text_file.write(" - `" + str(author) + "`_") + text_file.write("\n") + + # shepherd (currnely only one) shepherd = re.search('Shepherd: (.+?)\n', str(item['body'])) if shepherd is not None: authors.add(shepherd.group(1).strip()) From 3b4e56c4e085bc33bd9ae58c870c4f6558699b8d Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Mon, 14 May 2018 09:48:54 +0100 Subject: [PATCH 016/308] handle dd/mm/yyyy and yyyy-mm-dd dates in text --- scripts/proposals.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/proposals.py b/scripts/proposals.py index c3d65678..a27961ff 100644 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -74,6 +74,18 @@ for label in labels: created = re.search('^Date: (.+?)\n', str(item['body']), flags=re.MULTILINE) if created is not None: created = created.group(1).strip() + try: + created = datetime.strptime(created, "%d/%m/%Y") + created = created.strftime('%Y-%m-%d') + except: + pass + + try: + created = datetime.strptime(created, "%Y-%m-%d") + created = created.strftime('%Y-%m-%d') + except: + pass + else : created = datetime.strptime(item['created_at'], "%Y-%m-%dT%XZ") created = created.strftime('%Y-%m-%d') From 867307af8be08fae5420b58e3beaba88b0e3656b Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Mon, 14 May 2018 09:57:43 +0100 Subject: [PATCH 017/308] new flow --- specification/proposals_intro.rst | 55 ++++++++++++++++--------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index 3f3f5caa..d5687d01 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -26,39 +26,40 @@ We use labels and some metadata in the issue text to generate this page. When ad +-------+ | +------+ | +----------+ | | | | - +----------+ | +------------------+ | +---------+ - | | | | | | | | - | Proposal | | +-> Spec PR | | | Blocked | - | WIP | | | | Ready for Review | | | | - | | | | | | | +---------+ - +----+-----+ | | +---------+--------+ | + +----------+ | +---------+ | +---------+ + | | | | | | | | + | Proposal | | +------> Spec PR | | | Blocked | + | WIP | | | | Missing | | | | + | | | | | | | +---------+ + +----+-----+ | | +----+----+ | | | | | | | | | | | +-----------+ - +--------v----------+ | | +-----v-----+ | | | - | | | | | | | | Abandoned | - | Proposal | | | | Spec PR | | | | - | Ready for Review | | | | In Review | | +-----------+ - | | | | | | | - +----------+--------+ | | +-----+-----+ | +-----------+ - | | | | | | | + +--------v----------+ | | | | | | + | | | | +---------v--------+ | | Abandoned | + | Proposal | | | | | | | | + | Ready for Review | | | | Spec PR | | +-----------+ + | | | | | Ready for Review | | + +----------+--------+ | | | | | +-----------+ + | | | +---------+--------+ | | | | | | | | | Obsolete | +------v----+ | | | | | | - | | | | +-----------+ | +-----------+ - | Proposal | | | |-----v-----| | - | In Review | | | || || | - | | | | || Merged || | +----------+ - +----+------+ | | || || | | | - | | | |-----------| | | Rejected | - | | | +-----------+ | | | - +------v--------+ | | | +----------+ - | | | | | - | Proposal | | | | - | Passed Review | | | | - | | | | | - +-------+-------+ | | | - | | | | + | | | | +-----v-----+ | +-----------+ + | Proposal | | | | | | + | In Review | | | | Spec PR | | + | | | | | In Review | | +----------+ + +----+------+ | | | | | | | + | | | +-----+-----+ | | Rejected | + | | | | | | | + +------v--------+ | | | | +----------+ + | | | | | | + | Proposal | | | +----v----+ | + | Passed Review | | | | | | + | | | | | Merged! | | + +-------+-------+ | | | | | + | | | +---------+ | | | | | +---------------+ | | | + + + From 120fa92078f28202b949e0068893fbd78f30dbff Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Mon, 14 May 2018 10:25:10 +0100 Subject: [PATCH 018/308] update intro --- specification/proposals_intro.rst | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index d5687d01..c87f5d1d 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -10,14 +10,17 @@ Proposal Tracking This is a living document generated from the list of proposals at `matrix-doc/issues `_ on GitHub. -We use labels and some metadata in the issue text to generate this page. When adding or updating an issue, make sure you specify the current status as a label: +We use labels and some metadata in the issue text to generate this page. When adding or updating an issue, make sure you specify the current status as a label per the diagram below, these labels already exist in the matrix-doc repo. + +Other metadata: + +- the MSC (Matrix Spec Change) number is taken from the github issue ID. This is carried for the lifetime of the proposal, including the PR creation phase. +- Please use the github issue title to set the title. +- The created date is taken from the github issue, but can be overriden by adding a "Date: yyyy-mm-dd" line in the body of the issue text. +- Updated Date is taken from github. +- Author is the creator of the github issue, but can be overriden by adding a "Author: @username" line in the body of the issue text. Please make sure @username is a github user (include the @!) +- Shepherd is set by adding a "Shepherd: @username" line in the body of the issue text. Again, make sure this is a real Github user. -- WIP -- Ready for review -- In review -- Merged -- Rejected -- Stalled :: From e87f1f26198a14d837a0bfe3a0953fb8ca052c3b Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Mon, 14 May 2018 11:32:40 +0100 Subject: [PATCH 019/308] include PRs in tables --- scripts/proposals.py | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index a27961ff..aa9ba5d3 100644 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -12,6 +12,7 @@ from m2r import convert as m2r pagecount = 1 authors = set() +prs = set() def getpage(url, page): resp = requests.get(url + str(page)) @@ -35,8 +36,8 @@ def getbylabel(label): # new status labels: labels = ['proposal-wip', 'proposal-ready-for-review', - 'proposal-in-review', 'proposal-passed-review', - 'spec-pr-ready-for-review', 'spec-pr-in-review', 'merged', 'abandoned', 'rejected', 'blocked' ] + 'proposal-in-review', 'proposal-passed-review', 'spec-pr-missing', + 'spec-pr-ready-for-review', 'spec-pr-in-review', 'merged', 'abandoned', 'rejected', 'blocked', 'obsolete' ] #labels = ['p1', 'p2', 'p3', 'p4', 'p5'] issues = {} @@ -56,12 +57,13 @@ for label in labels: text_file.write(label + "\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n") text_file.write(".. list-table::\n :header-rows: 1\n :widths: auto\n :stub-columns: 1\n\n") text_file.write(" * - MSC\n") - text_file.write(" - proposal title\n") - text_file.write(" - created_at\n") - text_file.write(" - updated_at\n") - text_file.write(" - maindoc\n") - text_file.write(" - author\n") - text_file.write(" - shepherd\n") + text_file.write(" - Proposal Title\n") + text_file.write(" - Creation Date\n") + text_file.write(" - Update Date\n") + text_file.write(" - Documenation\n") + text_file.write(" - Author\n") + text_file.write(" - Shepherd\n") + text_file.write(" - PRs\n") for item in issues[label]: # MSC number @@ -126,6 +128,20 @@ for label in labels: authors.add(shepherd.group(1).strip()) shepherd = "`" + shepherd.group(1).strip() + "`_" text_file.write(" - " + str(shepherd) + "\n") + + # PRs + pr_list = re.search('PRs: (.+?)\n', str(item['body'])) + if pr_list is not None: + pr_list_formatted = set() + pr_list = pr_list.group(1) + for p in pr_list.split(","): + prs.add(p.strip()) + pr_list_formatted.add("`PR" + str(p.strip()) + "`_") + text_file.write(" - " + ', '.join(pr_list_formatted)) + text_file.write("\n") + else: + text_file.write(" - \n") + text_file.write("\n\n\n") text_file.write("\n") @@ -133,4 +149,7 @@ text_file.write("\n") for author in authors: text_file.write("\n.. _" + author + ": https://github.com/" + author[1:]) +for pr in prs: + text_file.write("\n.. _PR" + pr + ": https://github.com/matrix-org/matrix-doc/pull/" + pr) + text_file.close() From b8f38b21a7627adfccdb868416aaa23fd55ee075 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Mon, 14 May 2018 14:25:12 +0100 Subject: [PATCH 020/308] update introductory text --- scripts/proposals.py | 2 +- specification/proposals_intro.rst | 35 +++++++++++++++++-------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index aa9ba5d3..0b0717d9 100644 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -47,7 +47,7 @@ for label in labels: text_file = open("../specification/proposals.rst", "w") -text_file.write("Tables\n------------------\n\n") +text_file.write("Tables of Tracked Proposals\n---------------------------\n\n") for label in labels: diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index c87f5d1d..d6d79c92 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -3,24 +3,13 @@ Proposals for Spec Changes to Matrix ------------------------------------ +The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: +- produce a publicly-accessible Google Doc describing you change +- make a new issue at https://github.com/matrix-org/matrix-doc/issues with the metadata appropriate to the section below +- make a new PR which includes the proposed changed at https://github.com/matrix-org/matrix-doc/tree/master/specification -Proposal Tracking ------------------ - -This is a living document generated from the list of proposals at `matrix-doc/issues `_ on GitHub. - -We use labels and some metadata in the issue text to generate this page. When adding or updating an issue, make sure you specify the current status as a label per the diagram below, these labels already exist in the matrix-doc repo. - -Other metadata: - -- the MSC (Matrix Spec Change) number is taken from the github issue ID. This is carried for the lifetime of the proposal, including the PR creation phase. -- Please use the github issue title to set the title. -- The created date is taken from the github issue, but can be overriden by adding a "Date: yyyy-mm-dd" line in the body of the issue text. -- Updated Date is taken from github. -- Author is the creator of the github issue, but can be overriden by adding a "Author: @username" line in the body of the issue text. Please make sure @username is a github user (include the @!) -- Shepherd is set by adding a "Shepherd: @username" line in the body of the issue text. Again, make sure this is a real Github user. - +The process for handling proposals is described in the following diagram. Note that there are corresponding labels for each stage available in the matrix-doc issue tracker. :: @@ -65,4 +54,18 @@ Other metadata: | | + + +Proposal Tracking +----------------- +This is a living document generated from the list of proposals at `matrix-doc/issues `_ on GitHub. + +We use labels and some metadata in the issue text to generate this page. When adding or updating an issue, make sure you specify the current status as a label per the diagram above, these labels already exist in the matrix-doc repo. + +Other metadata: + +- the MSC (Matrix Spec Change) number is taken from the github issue ID. This is carried for the lifetime of the proposal, including the PR creation phase. +- Please use the github issue title to set the title. +- The created date is taken from the github issue, but can be overriden by adding a "Date: yyyy-mm-dd" line in the body of the issue text. +- Updated Date is taken from github. +- Author is the creator of the github issue, but can be overriden by adding a "Author: @username" line in the body of the issue text. Please make sure @username is a github user (include the @!) +- Shepherd is set by adding a "Shepherd: @username" line in the body of the issue text. Again, make sure this is a real Github user. From 34140a785d52475c8898a5b9de9985b64eaec3cd Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Mon, 14 May 2018 16:02:48 +0200 Subject: [PATCH 021/308] fix typo in delete device --- api/client-server/device_management.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/device_management.yaml b/api/client-server/device_management.yaml index 2f7d322b..030ed831 100644 --- a/api/client-server/device_management.yaml +++ b/api/client-server/device_management.yaml @@ -139,7 +139,7 @@ paths: description: |- This API endpoint uses the `User-Interactive Authentication API`_. - Deletes the given device, and invalidates any access token assoicated with it. + Deletes the given device, and invalidates any access token associated with it. operationId: deleteDevice security: - accessToken: [] From 8c469e51cbf01caca3f2551e367dcd74de66d36d Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 14 May 2018 22:05:28 +0100 Subject: [PATCH 022/308] lots of tweaks to the MSC verbiage --- specification/proposals_intro.rst | 47 +++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index d6d79c92..f1a3adb6 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -3,13 +3,23 @@ Proposals for Spec Changes to Matrix ------------------------------------ + The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: -- produce a publicly-accessible Google Doc describing you change -- make a new issue at https://github.com/matrix-org/matrix-doc/issues with the metadata appropriate to the section below -- make a new PR which includes the proposed changed at https://github.com/matrix-org/matrix-doc/tree/master/specification +- produce a publicly-accessible proposal describing your change: + - Please use Google Docs, or an equivalent system capable of collaborative editing, with versioned history and threaded comments. + - We do not use Github issues (or Etherpad) for the design process of the proposal, as the document review/commenting capabilities aren't good enough. + - We also don't jump straight to PRing against the spec itself, as it's much faster to iterate on a proposal in freeform document form than in the terse and formal structure of the spec. + - In the proposal, please clearly state the problem being solved, and the possible solutions being proposed for solving it and their trade-offs. + - Proposal documents are intended to be as lightweight and flexible as the author desires; there is no formal template; the intention is to iterate as quickly as possible to get to a good design. +- make a new issue at https://github.com/matrix-org/matrix-doc/issues, whose description should list the metadata as per below. +- Gather feedback as widely as possible from the community and core team on the proposal. + - The aim is to get maximum consensus on the trade-offs chosen to get an optimal solution. + - Iterating on the proposal and getting consensus can sometimes be time-consuming; an impartial 'shepherd' can be assigned to help guide the proposal through this process. +- Once the proposal has consensus and passed review, you **must** show an implementation to prove that it works well in practice, before before a spec PR will be accepted. Iterate on the proposal if needed. +- Finally, please make a new spec PR which includes the proposed changes against https://github.com/matrix-org/matrix-doc/tree/master/specification. This will then be reviewed and hopefully merged! -The process for handling proposals is described in the following diagram. Note that there are corresponding labels for each stage available in the matrix-doc issue tracker. +The process for handling proposals is described in the following diagram. Note that the lifetime of a proposal is tracked through the corresponding labels for each stage in the `matrix-doc issue tracker `_. :: @@ -54,18 +64,37 @@ The process for handling proposals is described in the following diagram. Note t | | + + +Lifetime States +--------------- + +=========================== ======================================================= +Proposal WIP A proposal document which is still work-in-progress but is being shared to incorporate feedback +Proposal Ready for Review A proposal document which is now ready and waiting for review by the core team and community +Proposal In Review A proposal document which is currently in review +Proposal Passed Review A proposal document which has passed review as worth implementing and then being added to the spec +Spec PR Missing A proposal which has been implemented and is being used in the wild but hasn't yet been added to the spec +Spec PR Ready for Review A proposal which has been PR'd against the spec and is awaiting review +Spec PR In Review A proposal which has been PR'd against the spec and is in review +Merged A proposal whose PR has merged into the spec! +Blocked A proposal which is temporarily blocked on some external factor (e.g. being blocked on another proposal first being approved) +Abandoned A proposal where the author/shepherd is not responsive +Obsolete A proposal which has been overtaken by other proposals +Rejected A proposal which is not going to be incorporated into Matrix +=========================== ======================================================= + + Proposal Tracking ----------------- -This is a living document generated from the list of proposals at `matrix-doc/issues `_ on GitHub. +This is a living document generated from the list of proposals at `matrix-doc/issues `_ on GitHub. -We use labels and some metadata in the issue text to generate this page. When adding or updating an issue, make sure you specify the current status as a label per the diagram above, these labels already exist in the matrix-doc repo. +We use labels and some metadata in the issue's description to generate this page. Labels are assigned by the core team whilst triaging the issues based on those which exist in the matrix-doc repo already. Other metadata: -- the MSC (Matrix Spec Change) number is taken from the github issue ID. This is carried for the lifetime of the proposal, including the PR creation phase. +- the MSC (Matrix Spec Change) number is taken from the github issue ID. This is carried for the lifetime of the proposal, including the PR creation phase. N.B. They are not in chronological order! - Please use the github issue title to set the title. -- The created date is taken from the github issue, but can be overriden by adding a "Date: yyyy-mm-dd" line in the body of the issue text. +- The created date is taken from the github issue, but can be overriden by adding a "Date: yyyy-mm-dd" line in the body of the issue text. - Updated Date is taken from github. - Author is the creator of the github issue, but can be overriden by adding a "Author: @username" line in the body of the issue text. Please make sure @username is a github user (include the @!) -- Shepherd is set by adding a "Shepherd: @username" line in the body of the issue text. Again, make sure this is a real Github user. +- Shepherd is set by adding a "Shepherd: @username" line in the issue description. Again, make sure this is a real Github user. From cb882ba12d16c5fdae98706d5487a91745a5e78b Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 14 May 2018 22:07:49 +0100 Subject: [PATCH 023/308] more tweaks --- specification/proposals_intro.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index f1a3adb6..6bdc653a 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -94,7 +94,8 @@ Other metadata: - the MSC (Matrix Spec Change) number is taken from the github issue ID. This is carried for the lifetime of the proposal, including the PR creation phase. N.B. They are not in chronological order! - Please use the github issue title to set the title. -- The created date is taken from the github issue, but can be overriden by adding a "Date: yyyy-mm-dd" line in the body of the issue text. +- Please link to the proposal document by adding a "Documentation: " line in the issue description. +- The creation date is taken from the github issue, but can be overriden by adding a "Date: yyyy-mm-dd" line in the issue description. - Updated Date is taken from github. -- Author is the creator of the github issue, but can be overriden by adding a "Author: @username" line in the body of the issue text. Please make sure @username is a github user (include the @!) -- Shepherd is set by adding a "Shepherd: @username" line in the issue description. Again, make sure this is a real Github user. +- Author is the creator of the github issue, but can be overriden by adding a "Author: @username" line in the body of the issue description. Please make sure @username is a github user (include the @!) +- A shepherd can be assigned by adding a "Shepherd: @username" line in the issue description. Again, make sure this is a real Github user. From 93681458d691fc978181dd4839e9217e563eb0a1 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 14 May 2018 22:13:28 +0100 Subject: [PATCH 024/308] more tweaks --- specification/proposals_intro.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index 6bdc653a..e1024e57 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -12,11 +12,11 @@ The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: - We also don't jump straight to PRing against the spec itself, as it's much faster to iterate on a proposal in freeform document form than in the terse and formal structure of the spec. - In the proposal, please clearly state the problem being solved, and the possible solutions being proposed for solving it and their trade-offs. - Proposal documents are intended to be as lightweight and flexible as the author desires; there is no formal template; the intention is to iterate as quickly as possible to get to a good design. -- make a new issue at https://github.com/matrix-org/matrix-doc/issues, whose description should list the metadata as per below. +- make a new issue at https://github.com/matrix-org/matrix-doc/issues (or modify an existing one), whose description should list the metadata as per below. - Gather feedback as widely as possible from the community and core team on the proposal. - The aim is to get maximum consensus on the trade-offs chosen to get an optimal solution. - - Iterating on the proposal and getting consensus can sometimes be time-consuming; an impartial 'shepherd' can be assigned to help guide the proposal through this process. -- Once the proposal has consensus and passed review, you **must** show an implementation to prove that it works well in practice, before before a spec PR will be accepted. Iterate on the proposal if needed. + - Iterating on the proposal and gathering consensus can sometimes be time-consuming; an impartial 'shepherd' can be assigned to help guide the proposal through this process. +- Once the proposal has sufficient consensus and passed review, you **must** show an implementation to prove that it works well in practice, before before a spec PR will be accepted. Iterate on the proposal if needed. - Finally, please make a new spec PR which includes the proposed changes against https://github.com/matrix-org/matrix-doc/tree/master/specification. This will then be reviewed and hopefully merged! The process for handling proposals is described in the following diagram. Note that the lifetime of a proposal is tracked through the corresponding labels for each stage in the `matrix-doc issue tracker `_. From 07bf61b33b00a87ee25069e722f11d90bb8af1a0 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 14 May 2018 22:22:09 +0100 Subject: [PATCH 025/308] more tweaks --- specification/proposals_intro.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index e1024e57..878faaa9 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -10,7 +10,7 @@ The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: - Please use Google Docs, or an equivalent system capable of collaborative editing, with versioned history and threaded comments. - We do not use Github issues (or Etherpad) for the design process of the proposal, as the document review/commenting capabilities aren't good enough. - We also don't jump straight to PRing against the spec itself, as it's much faster to iterate on a proposal in freeform document form than in the terse and formal structure of the spec. - - In the proposal, please clearly state the problem being solved, and the possible solutions being proposed for solving it and their trade-offs. + - In the proposal, please clearly state the problem being solved, and the possible solutions being proposed for solving it and their respective trade-offs. - Proposal documents are intended to be as lightweight and flexible as the author desires; there is no formal template; the intention is to iterate as quickly as possible to get to a good design. - make a new issue at https://github.com/matrix-org/matrix-doc/issues (or modify an existing one), whose description should list the metadata as per below. - Gather feedback as widely as possible from the community and core team on the proposal. @@ -19,6 +19,8 @@ The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: - Once the proposal has sufficient consensus and passed review, you **must** show an implementation to prove that it works well in practice, before before a spec PR will be accepted. Iterate on the proposal if needed. - Finally, please make a new spec PR which includes the proposed changes against https://github.com/matrix-org/matrix-doc/tree/master/specification. This will then be reviewed and hopefully merged! +Final decisions on review are made by the +matrix:matrix.org community (i.e. the core team), acting on behalf of the whole Matrix community. + The process for handling proposals is described in the following diagram. Note that the lifetime of a proposal is tracked through the corresponding labels for each stage in the `matrix-doc issue tracker `_. :: From 755c9473fb3b7d3341739eb50d4af0f954b4c90a Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 14 May 2018 22:27:43 +0100 Subject: [PATCH 026/308] more tweaks --- specification/proposals_intro.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index 878faaa9..b8cad362 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -16,8 +16,8 @@ The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: - Gather feedback as widely as possible from the community and core team on the proposal. - The aim is to get maximum consensus on the trade-offs chosen to get an optimal solution. - Iterating on the proposal and gathering consensus can sometimes be time-consuming; an impartial 'shepherd' can be assigned to help guide the proposal through this process. -- Once the proposal has sufficient consensus and passed review, you **must** show an implementation to prove that it works well in practice, before before a spec PR will be accepted. Iterate on the proposal if needed. -- Finally, please make a new spec PR which includes the proposed changes against https://github.com/matrix-org/matrix-doc/tree/master/specification. This will then be reviewed and hopefully merged! +- Once the proposal has sufficient consensus and passed review, you **must** show an implementation to prove that it works well in practice, before a spec PR will be accepted. Iterate on the proposal if needed. +- Finally, please make a new spec PR which includes the changes as implemented against https://github.com/matrix-org/matrix-doc/tree/master/specification. This will then be reviewed and hopefully merged! Final decisions on review are made by the +matrix:matrix.org community (i.e. the core team), acting on behalf of the whole Matrix community. From 714767c95a99cc7929266d402ec56460564a4466 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 14 May 2018 22:32:00 +0100 Subject: [PATCH 027/308] explain proposal neutrality --- specification/proposals_intro.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index b8cad362..ab1e4027 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -21,6 +21,10 @@ The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: Final decisions on review are made by the +matrix:matrix.org community (i.e. the core team), acting on behalf of the whole Matrix community. +Proposals **must** act to the greater benefit of the entire Matrix ecosystem, rather than benefiting or privileging any single player or subset of players - and must not contain any patent encumbered IP. The Matrix core team pledges to act as a neutral custodian for Matrix on behalf of the whole ecosystem. + +The directions above are intended to be short, simple and pragmatic - although a full governance document for the Matrix core team will follow in due course. + The process for handling proposals is described in the following diagram. Note that the lifetime of a proposal is tracked through the corresponding labels for each stage in the `matrix-doc issue tracker `_. :: From 29348c61906a688ed9573081e7f7e807a1254342 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 14 May 2018 22:35:53 +0100 Subject: [PATCH 028/308] improve wording --- specification/proposals_intro.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index ab1e4027..aaae7486 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -23,7 +23,7 @@ Final decisions on review are made by the +matrix:matrix.org community (i.e. the Proposals **must** act to the greater benefit of the entire Matrix ecosystem, rather than benefiting or privileging any single player or subset of players - and must not contain any patent encumbered IP. The Matrix core team pledges to act as a neutral custodian for Matrix on behalf of the whole ecosystem. -The directions above are intended to be short, simple and pragmatic - although a full governance document for the Matrix core team will follow in due course. +The directions above are not intended to be exhaustive but simple and pragmatic. A full governance document for the Matrix core team will follow in due course. The process for handling proposals is described in the following diagram. Note that the lifetime of a proposal is tracked through the corresponding labels for each stage in the `matrix-doc issue tracker `_. From e27f674fb95e0162d9b96e269e2d5b4aee4adcfa Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 14 May 2018 23:00:54 +0100 Subject: [PATCH 029/308] incept dates --- specification/proposals_intro.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index aaae7486..c865f6af 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -21,7 +21,7 @@ The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: Final decisions on review are made by the +matrix:matrix.org community (i.e. the core team), acting on behalf of the whole Matrix community. -Proposals **must** act to the greater benefit of the entire Matrix ecosystem, rather than benefiting or privileging any single player or subset of players - and must not contain any patent encumbered IP. The Matrix core team pledges to act as a neutral custodian for Matrix on behalf of the whole ecosystem. +Proposals **must** act to the greater benefit of the entire Matrix ecosystem, rather than benefiting or privileging any single player or subset of players - and must not contain any patent encumbered IP. The Matrix core team pledges to act as a neutral custodian for Matrix on behalf of the whole ecosystem, just as it has since Matrix's inception in May 2014. The directions above are not intended to be exhaustive but simple and pragmatic. A full governance document for the Matrix core team will follow in due course. @@ -101,6 +101,7 @@ Other metadata: - the MSC (Matrix Spec Change) number is taken from the github issue ID. This is carried for the lifetime of the proposal, including the PR creation phase. N.B. They are not in chronological order! - Please use the github issue title to set the title. - Please link to the proposal document by adding a "Documentation: " line in the issue description. +- Please link to the spec PR (if any) by adding a "PRs: #1234" line in the issue description. - The creation date is taken from the github issue, but can be overriden by adding a "Date: yyyy-mm-dd" line in the issue description. - Updated Date is taken from github. - Author is the creator of the github issue, but can be overriden by adding a "Author: @username" line in the body of the issue description. Please make sure @username is a github user (include the @!) From f1cc26dfd7f266363bcf8f2b32ec8a375ca12225 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Tue, 15 May 2018 11:59:34 +0100 Subject: [PATCH 030/308] sort output by Created date per-table --- scripts/proposals.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index 0b0717d9..1e556ccb 100644 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -43,7 +43,6 @@ issues = {} for label in labels: issues[label] = getbylabel(label) - print(issues) text_file = open("../specification/proposals.rst", "w") @@ -66,14 +65,10 @@ for label in labels: text_file.write(" - PRs\n") for item in issues[label]: - # MSC number - text_file.write(" * - `MSC" + str(item['number']) + " <" + item['html_url'] + ">`_\n") - - # title from Github issue - text_file.write(" - " + item['title'] + "\n") - - # created date, find local field, otherwise Github - created = re.search('^Date: (.+?)\n', str(item['body']), flags=re.MULTILINE) + # set the created date, find local field, otherwise Github + print(item) + body = str(item['body']) + created = re.search('^Date: (.+?)\n', body, flags=re.MULTILINE) if created is not None: created = created.group(1).strip() try: @@ -81,17 +76,28 @@ for label in labels: created = created.strftime('%Y-%m-%d') except: pass - try: created = datetime.strptime(created, "%Y-%m-%d") created = created.strftime('%Y-%m-%d') except: pass - else : created = datetime.strptime(item['created_at'], "%Y-%m-%dT%XZ") created = created.strftime('%Y-%m-%d') - text_file.write(" - " + created + "\n") + item['created'] = created + + issues_to_print = list(issues[label]) + issues_to_print.sort(key=lambda issue_sort: issue_sort["created"]) + + for item in issues_to_print: + # MSC number + text_file.write(" * - `MSC" + str(item['number']) + " <" + item['html_url'] + ">`_\n") + + # title from Github issue + text_file.write(" - " + item['title'] + "\n") + + # created date + text_file.write(" - " + item['created'] + "\n") # last updated, purely Github updated = datetime.strptime(item['updated_at'], "%Y-%m-%dT%XZ") From 3cef79f31d92b479b8fa3a0aa0071d4974a4029a Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Tue, 15 May 2018 12:09:35 +0100 Subject: [PATCH 031/308] sublist formatting --- specification/proposals_intro.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index c865f6af..3fc7470e 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -7,15 +7,19 @@ Proposals for Spec Changes to Matrix The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: - produce a publicly-accessible proposal describing your change: + - Please use Google Docs, or an equivalent system capable of collaborative editing, with versioned history and threaded comments. - We do not use Github issues (or Etherpad) for the design process of the proposal, as the document review/commenting capabilities aren't good enough. - We also don't jump straight to PRing against the spec itself, as it's much faster to iterate on a proposal in freeform document form than in the terse and formal structure of the spec. - In the proposal, please clearly state the problem being solved, and the possible solutions being proposed for solving it and their respective trade-offs. - Proposal documents are intended to be as lightweight and flexible as the author desires; there is no formal template; the intention is to iterate as quickly as possible to get to a good design. + - make a new issue at https://github.com/matrix-org/matrix-doc/issues (or modify an existing one), whose description should list the metadata as per below. - Gather feedback as widely as possible from the community and core team on the proposal. + - The aim is to get maximum consensus on the trade-offs chosen to get an optimal solution. - Iterating on the proposal and gathering consensus can sometimes be time-consuming; an impartial 'shepherd' can be assigned to help guide the proposal through this process. + - Once the proposal has sufficient consensus and passed review, you **must** show an implementation to prove that it works well in practice, before a spec PR will be accepted. Iterate on the proposal if needed. - Finally, please make a new spec PR which includes the changes as implemented against https://github.com/matrix-org/matrix-doc/tree/master/specification. This will then be reviewed and hopefully merged! From 3b736388ce82736ee817ff053237c65bc6be281b Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 15 May 2018 14:17:30 +0100 Subject: [PATCH 032/308] clarify governance --- specification/proposals_intro.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index 3fc7470e..fd692e52 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -27,7 +27,7 @@ Final decisions on review are made by the +matrix:matrix.org community (i.e. the Proposals **must** act to the greater benefit of the entire Matrix ecosystem, rather than benefiting or privileging any single player or subset of players - and must not contain any patent encumbered IP. The Matrix core team pledges to act as a neutral custodian for Matrix on behalf of the whole ecosystem, just as it has since Matrix's inception in May 2014. -The directions above are not intended to be exhaustive but simple and pragmatic. A full governance document for the Matrix core team will follow in due course. +The above directions are intended to be simple and pragmatic rather than exhaustive, and aim to provide guidelinnes until we have a formal spec governance process in place that covers the whole Matrix community. In order to get Matrix out of beta as quickly as possible, as of May 2018 we are prioritising spec and reference implementation development over writing formal governance, but a formal governance document will follow as rapidly as that allows. The process for handling proposals is described in the following diagram. Note that the lifetime of a proposal is tracked through the corresponding labels for each stage in the `matrix-doc issue tracker `_. From 42fd3f34e4daf8edcde68106f285e34ee0f9cb6f Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 15 May 2018 14:46:08 +0100 Subject: [PATCH 033/308] shift stuff from contributing.rst to the new proposals page --- CONTRIBUTING.rst | 43 +++++-------------------------- specification/proposals_intro.rst | 4 +-- 2 files changed, 9 insertions(+), 38 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 6f17afe6..2a7416b1 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -21,44 +21,15 @@ Matrix-doc workflows Specification changes ~~~~~~~~~~~~~~~~~~~~~ -The Matrix specification documents the APIs which Matrix clients can use. For -this to be effective, the APIs need to be present and working correctly in a -server before they can be documented in the specification. This process can -take some time to complete. +The Matrix specification documents the APIs which Matrix clients and servers use. +For this to be effective, the APIs need to be present and working correctly in a +server before they can be documented in the specification. This process can take +some time to complete. For this reason, we have not found the github pull-request model effective for -discussing changes to the specification. Instead, we have adopted the following -workflow: - -1. Create a discussion document outlining the proposed change. The document - should include details such as the HTTP endpoint being changed (or the - suggested URL for a new endpoint), any new or changed parameters and response - fields, and generally as much detail about edge-cases and error handling as - is practical at this stage. - - The Matrix Core Team's preferred tool for such discussion documents is - `Google Docs `_ thanks to its support for comment - threads. Works in progress are kept in the `Matrix Design drafts folder - `_. - -2. Seek feedback on the proposal. `#matrix-dev:matrix.org - `_ is a good place to reach the - core team and others who may be interested in your proposal. - -3. Implement the changes in servers and clients. Refer to the CONTRIBUTING files - of the relevant projects for details of how best to do this. - - In general we will be unable to publish specification updates until the - reference server implements them, and they have been proven by a working - client implementation. - -4. Iterate steps 1-3 as necessary. - -5. Write the specification for the change, and create a `pull request`_ for - it. It may be that much of the text of the change can be taken directly from - the discussion document, though typically some effort will be needed to - change to the ReST syntax and to ensure that the text is as clear as - possible. +discussing changes to the specification. Instead, we have adopted the workflow +as described at https://matrix.org/docs/spec/proposals - *please read this for +details on how to contribute spec changes*. Other changes diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index fd692e52..e1c467ca 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -21,9 +21,9 @@ The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: - Iterating on the proposal and gathering consensus can sometimes be time-consuming; an impartial 'shepherd' can be assigned to help guide the proposal through this process. - Once the proposal has sufficient consensus and passed review, you **must** show an implementation to prove that it works well in practice, before a spec PR will be accepted. Iterate on the proposal if needed. -- Finally, please make a new spec PR which includes the changes as implemented against https://github.com/matrix-org/matrix-doc/tree/master/specification. This will then be reviewed and hopefully merged! +- Finally, please make a new spec PR which includes the changes as implemented against https://github.com/matrix-org/matrix-doc/tree/master/specification. This will then be reviewed and hopefully merged! Please sign off the spec PR as per the `CONTRIBUTING.rst `_ guidelines. -Final decisions on review are made by the +matrix:matrix.org community (i.e. the core team), acting on behalf of the whole Matrix community. +Final decisions on review are made by the Matrix core team (+matrix:matrix.org), acting on behalf of the whole Matrix community. Proposals **must** act to the greater benefit of the entire Matrix ecosystem, rather than benefiting or privileging any single player or subset of players - and must not contain any patent encumbered IP. The Matrix core team pledges to act as a neutral custodian for Matrix on behalf of the whole ecosystem, just as it has since Matrix's inception in May 2014. From 3e10a5a24a81180c89aa411c0c60db22f2850bf0 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 15 May 2018 15:02:35 +0100 Subject: [PATCH 034/308] enter #matrix-spec:matrix.org --- specification/proposals_intro.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index e1c467ca..019d8eab 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -18,6 +18,7 @@ The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: - Gather feedback as widely as possible from the community and core team on the proposal. - The aim is to get maximum consensus on the trade-offs chosen to get an optimal solution. + - A good place to ask for feedback on a specific proposal is `#matrix-spec:matrix.org `_. However, authors/shepherds are welcome to use an alternative room if they prefer - please advertise it in #matrix-spec:matrix.org though and link to it on the github issue. N.B. that #matrix-dev:matrix.org is for developers using existing Matrix APIs, #matrix:matrix.org is for users trying to run matrix apps (clients & servers); #matrix-architecture:matrix.org is for cross-cutting discussion of Matrix's architecture. - Iterating on the proposal and gathering consensus can sometimes be time-consuming; an impartial 'shepherd' can be assigned to help guide the proposal through this process. - Once the proposal has sufficient consensus and passed review, you **must** show an implementation to prove that it works well in practice, before a spec PR will be accepted. Iterate on the proposal if needed. From 8096bf0039df0c7711d5c528ec144ee6e396d5de Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 15 May 2018 15:07:47 +0100 Subject: [PATCH 035/308] perms and apostrophes --- specification/proposals_intro.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index 019d8eab..35d0adbf 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -8,7 +8,7 @@ The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: - produce a publicly-accessible proposal describing your change: - - Please use Google Docs, or an equivalent system capable of collaborative editing, with versioned history and threaded comments. + - Please use Google Docs, or an equivalent system capable of collaborative editing, with versioned history and threaded comments. Please ensure the document is world-commentable or -editable. - We do not use Github issues (or Etherpad) for the design process of the proposal, as the document review/commenting capabilities aren't good enough. - We also don't jump straight to PRing against the spec itself, as it's much faster to iterate on a proposal in freeform document form than in the terse and formal structure of the spec. - In the proposal, please clearly state the problem being solved, and the possible solutions being proposed for solving it and their respective trade-offs. @@ -99,7 +99,7 @@ Proposal Tracking This is a living document generated from the list of proposals at `matrix-doc/issues `_ on GitHub. -We use labels and some metadata in the issue's description to generate this page. Labels are assigned by the core team whilst triaging the issues based on those which exist in the matrix-doc repo already. +We use labels and some metadata in the issues' descriptions to generate this page. Labels are assigned by the core team whilst triaging the issues based on those which exist in the matrix-doc repo already. Other metadata: From 4cdb4f105cfe4ad247977b7a7761b7947c8ceccc Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 15 May 2018 15:11:51 +0100 Subject: [PATCH 036/308] clarify architecture --- specification/proposals_intro.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index 35d0adbf..fb72836c 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -18,7 +18,7 @@ The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: - Gather feedback as widely as possible from the community and core team on the proposal. - The aim is to get maximum consensus on the trade-offs chosen to get an optimal solution. - - A good place to ask for feedback on a specific proposal is `#matrix-spec:matrix.org `_. However, authors/shepherds are welcome to use an alternative room if they prefer - please advertise it in #matrix-spec:matrix.org though and link to it on the github issue. N.B. that #matrix-dev:matrix.org is for developers using existing Matrix APIs, #matrix:matrix.org is for users trying to run matrix apps (clients & servers); #matrix-architecture:matrix.org is for cross-cutting discussion of Matrix's architecture. + - A good place to ask for feedback on a specific proposal is `#matrix-spec:matrix.org `_. However, authors/shepherds are welcome to use an alternative room if they prefer - please advertise it in #matrix-spec:matrix.org though and link to it on the github issue. N.B. that #matrix-dev:matrix.org is for developers using existing Matrix APIs, #matrix:matrix.org is for users trying to run matrix apps (clients & servers); #matrix-architecture:matrix.org is for cross-cutting discussion of Matrix's architectural design. - Iterating on the proposal and gathering consensus can sometimes be time-consuming; an impartial 'shepherd' can be assigned to help guide the proposal through this process. - Once the proposal has sufficient consensus and passed review, you **must** show an implementation to prove that it works well in practice, before a spec PR will be accepted. Iterate on the proposal if needed. From d338f189278b398da6cc185a73ab8051abc7c5ce Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Tue, 15 May 2018 15:22:03 +0100 Subject: [PATCH 037/308] handle EOF gracefully when searching for Authors and PRs --- scripts/proposals.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index 1e556ccb..e4c65daa 100644 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -1,9 +1,5 @@ # proposals.py: generate an RST file (proposals.rst) from queries to github.com/matrix.org/matrix-doc/issues. # v0.0.1 -# todo: -# use status labels to create separate sections. -# include all fields discussed at https://docs.google.com/document/d/1wLln7da12l0H5YgAh5xM2TVE7VsTjXzhEwVh3sRBMCk/edit# -# format, probably as tables import requests import re @@ -114,7 +110,7 @@ for label in labels: text_file.write("\n") # author list, if missing just use Github issue creator - author = re.search('^Author: (.+?)\n', str(item['body']), flags=re.MULTILINE) + author = re.search('^Author: (.+?)$', str(item['body']), flags=re.MULTILINE) if author is not None: author_list_formatted = set() author_list = author.group(1) @@ -136,7 +132,7 @@ for label in labels: text_file.write(" - " + str(shepherd) + "\n") # PRs - pr_list = re.search('PRs: (.+?)\n', str(item['body'])) + pr_list = re.search('PRs: (.+?)$', str(item['body'])) if pr_list is not None: pr_list_formatted = set() pr_list = pr_list.group(1) From 4d59abebf020d56e97594825b5608edaa77056dd Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Tue, 15 May 2018 15:33:39 +0100 Subject: [PATCH 038/308] handle PR links from GitHub --- scripts/proposals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index e4c65daa..93964143 100644 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -152,6 +152,6 @@ for author in authors: text_file.write("\n.. _" + author + ": https://github.com/" + author[1:]) for pr in prs: - text_file.write("\n.. _PR" + pr + ": https://github.com/matrix-org/matrix-doc/pull/" + pr) + text_file.write("\n.. _PR" + pr + ": https://github.com/matrix-org/matrix-doc/pull/" + pr.replace('#', '')) text_file.close() From 10a8cb3f67db4ed99e7e5ffffeb664f5bccc1c97 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 15 May 2018 15:44:40 +0100 Subject: [PATCH 039/308] put handwavey timings on state transitions --- specification/proposals_intro.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index fb72836c..a174ae6c 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -83,12 +83,12 @@ Proposal WIP A proposal document which is still work-in-progress Proposal Ready for Review A proposal document which is now ready and waiting for review by the core team and community Proposal In Review A proposal document which is currently in review Proposal Passed Review A proposal document which has passed review as worth implementing and then being added to the spec -Spec PR Missing A proposal which has been implemented and is being used in the wild but hasn't yet been added to the spec +Spec PR Missing A proposal which has been implemented and has been used in the wild for a few months but hasn't yet been added to the spec Spec PR Ready for Review A proposal which has been PR'd against the spec and is awaiting review Spec PR In Review A proposal which has been PR'd against the spec and is in review Merged A proposal whose PR has merged into the spec! Blocked A proposal which is temporarily blocked on some external factor (e.g. being blocked on another proposal first being approved) -Abandoned A proposal where the author/shepherd is not responsive +Abandoned A proposal where the author/shepherd has not been responsive for a few months Obsolete A proposal which has been overtaken by other proposals Rejected A proposal which is not going to be incorporated into Matrix =========================== ======================================================= From 4c0743ef65f17ae14e9ba50734c0399bde375c1e Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 15 May 2018 15:47:34 +0100 Subject: [PATCH 040/308] fix wording --- specification/proposals_intro.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index a174ae6c..34e6c58e 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -28,7 +28,7 @@ Final decisions on review are made by the Matrix core team (+matrix:matrix.org), Proposals **must** act to the greater benefit of the entire Matrix ecosystem, rather than benefiting or privileging any single player or subset of players - and must not contain any patent encumbered IP. The Matrix core team pledges to act as a neutral custodian for Matrix on behalf of the whole ecosystem, just as it has since Matrix's inception in May 2014. -The above directions are intended to be simple and pragmatic rather than exhaustive, and aim to provide guidelinnes until we have a formal spec governance process in place that covers the whole Matrix community. In order to get Matrix out of beta as quickly as possible, as of May 2018 we are prioritising spec and reference implementation development over writing formal governance, but a formal governance document will follow as rapidly as that allows. +The above directions are intended to be simple and pragmatic rather than exhaustive, and aim to provide guidelines until we have a formal spec governance process in place that covers the whole Matrix community. In order to get Matrix out of beta as quickly as possible, as of May 2018 we are prioritising spec and reference implementation development over writing formal governance, but a formal governance document will follow as rapidly as possible. The process for handling proposals is described in the following diagram. Note that the lifetime of a proposal is tracked through the corresponding labels for each stage in the `matrix-doc issue tracker `_. From 73b28612ec087633f08ec5ab3d5c7083e593da4e Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Tue, 15 May 2018 16:07:55 +0100 Subject: [PATCH 041/308] updates from feedback on pull --- scripts/proposals.py | 2 +- specification/proposals_intro.rst | 106 +++++++++++++++++++++++------- 2 files changed, 83 insertions(+), 25 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index 93964143..8b2107ac 100644 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -124,7 +124,7 @@ for label in labels: text_file.write(" - `" + str(author) + "`_") text_file.write("\n") - # shepherd (currnely only one) + # shepherd (currently only one) shepherd = re.search('Shepherd: (.+?)\n', str(item['body'])) if shepherd is not None: authors.add(shepherd.group(1).strip()) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index 34e6c58e..97b1ea22 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -8,29 +8,75 @@ The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: - produce a publicly-accessible proposal describing your change: - - Please use Google Docs, or an equivalent system capable of collaborative editing, with versioned history and threaded comments. Please ensure the document is world-commentable or -editable. - - We do not use Github issues (or Etherpad) for the design process of the proposal, as the document review/commenting capabilities aren't good enough. - - We also don't jump straight to PRing against the spec itself, as it's much faster to iterate on a proposal in freeform document form than in the terse and formal structure of the spec. - - In the proposal, please clearly state the problem being solved, and the possible solutions being proposed for solving it and their respective trade-offs. - - Proposal documents are intended to be as lightweight and flexible as the author desires; there is no formal template; the intention is to iterate as quickly as possible to get to a good design. + - Please use Google Docs, or an equivalent system capable of collaborative + editing, with versioned history and threaded comments. Please ensure the + document is world-commentable or -editable. + - We do not use Github issues (or Etherpad) for the design process of the + proposal, as the document review/commenting capabilities aren't good + enough. + - We also don't jump straight to PRing against the spec itself, as it's much + faster to iterate on a proposal in freeform document form than in the + terse and formal structure of the spec. + - In the proposal, please clearly state the problem being solved, and the + possible solutions being proposed for solving it and their respective + trade-offs. + - Proposal documents are intended to be as lightweight and flexible as the + author desires; there is no formal template; the intention is to iterate + as quickly as possible to get to a good design. -- make a new issue at https://github.com/matrix-org/matrix-doc/issues (or modify an existing one), whose description should list the metadata as per below. -- Gather feedback as widely as possible from the community and core team on the proposal. +- make a new issue at https://github.com/matrix-org/matrix-doc/issues (or + modify an existing one), whose description should list the metadata as per + below. +- Gather feedback as widely as possible from the community and core team on + the proposal. - - The aim is to get maximum consensus on the trade-offs chosen to get an optimal solution. - - A good place to ask for feedback on a specific proposal is `#matrix-spec:matrix.org `_. However, authors/shepherds are welcome to use an alternative room if they prefer - please advertise it in #matrix-spec:matrix.org though and link to it on the github issue. N.B. that #matrix-dev:matrix.org is for developers using existing Matrix APIs, #matrix:matrix.org is for users trying to run matrix apps (clients & servers); #matrix-architecture:matrix.org is for cross-cutting discussion of Matrix's architectural design. - - Iterating on the proposal and gathering consensus can sometimes be time-consuming; an impartial 'shepherd' can be assigned to help guide the proposal through this process. + - The aim is to get maximum consensus on the trade-offs chosen to get an + optimal solution. + - A good place to ask for feedback on a specific proposal is + `#matrix-spec:matrix.org `_. + However, authors/shepherds are welcome to use an alternative room if they + prefer - please advertise it in #matrix-spec:matrix.org though and link + to it on the github issue. N.B. that #matrix-dev:matrix.org is for + developers using existing Matrix APIs, #matrix:matrix.org is for users + trying to run matrix apps (clients & servers); + #matrix-architecture:matrix.org is for cross-cutting discussion of + Matrix's architectural design. + - Iterating on the proposal and gathering consensus can sometimes be + time-consuming; an impartial 'shepherd' can be assigned to help guide the + proposal through this process. -- Once the proposal has sufficient consensus and passed review, you **must** show an implementation to prove that it works well in practice, before a spec PR will be accepted. Iterate on the proposal if needed. -- Finally, please make a new spec PR which includes the changes as implemented against https://github.com/matrix-org/matrix-doc/tree/master/specification. This will then be reviewed and hopefully merged! Please sign off the spec PR as per the `CONTRIBUTING.rst `_ guidelines. +- Once the proposal has sufficient consensus and passed review, you **must** + show an implementation to prove that it works well in practice, before a + spec PR will be accepted. Iterate on the proposal if needed. +- Finally, please make a new spec PR which includes the changes as + implemented against + https://github.com/matrix-org/matrix-doc/tree/master/specification. This + will then be reviewed and hopefully merged! Please sign off the spec PR as + per the `CONTRIBUTING.rst + `_ + guidelines. -Final decisions on review are made by the Matrix core team (+matrix:matrix.org), acting on behalf of the whole Matrix community. +Final decisions on review are made by the Matrix core team +(+matrix:matrix.org), acting on behalf of the whole Matrix community. -Proposals **must** act to the greater benefit of the entire Matrix ecosystem, rather than benefiting or privileging any single player or subset of players - and must not contain any patent encumbered IP. The Matrix core team pledges to act as a neutral custodian for Matrix on behalf of the whole ecosystem, just as it has since Matrix's inception in May 2014. +Proposals **must** act to the greater benefit of the entire Matrix ecosystem, +rather than benefiting or privileging any single player or subset of players +- and must not contain any patent encumbered IP. The Matrix core team pledges +to act as a neutral custodian for Matrix on behalf of the whole ecosystem, +just as it has since Matrix's inception in May 2014. -The above directions are intended to be simple and pragmatic rather than exhaustive, and aim to provide guidelines until we have a formal spec governance process in place that covers the whole Matrix community. In order to get Matrix out of beta as quickly as possible, as of May 2018 we are prioritising spec and reference implementation development over writing formal governance, but a formal governance document will follow as rapidly as possible. +The above directions are intended to be simple and pragmatic rather than +exhaustive, and aim to provide guidelines until we have a formal spec +governance process in place that covers the whole Matrix community. In order +to get Matrix out of beta as quickly as possible, as of May 2018 we are +prioritising spec and reference implementation development over writing formal +governance, but a formal governance document will follow as rapidly as +possible. -The process for handling proposals is described in the following diagram. Note that the lifetime of a proposal is tracked through the corresponding labels for each stage in the `matrix-doc issue tracker `_. +The process for handling proposals is described in the following diagram. Note +that the lifetime of a proposal is tracked through the corresponding labels for +each stage in the `matrix-doc issue tracker +`_. :: @@ -97,17 +143,29 @@ Rejected A proposal which is not going to be incorporated int Proposal Tracking ----------------- -This is a living document generated from the list of proposals at `matrix-doc/issues `_ on GitHub. +This is a living document generated from the list of proposals at +`matrix-doc/issues `_ on +GitHub. -We use labels and some metadata in the issues' descriptions to generate this page. Labels are assigned by the core team whilst triaging the issues based on those which exist in the matrix-doc repo already. +We use labels and some metadata in the issues' descriptions to generate this +page. Labels are assigned by the core team whilst triaging the issues based +on those which exist in the matrix-doc repo already. Other metadata: -- the MSC (Matrix Spec Change) number is taken from the github issue ID. This is carried for the lifetime of the proposal, including the PR creation phase. N.B. They are not in chronological order! +- the MSC (Matrix Spec Change) number is taken from the github issue ID. This + is carried for the lifetime of the proposal, including the PR creation + phase. N.B. They are not in chronological order! - Please use the github issue title to set the title. -- Please link to the proposal document by adding a "Documentation: " line in the issue description. -- Please link to the spec PR (if any) by adding a "PRs: #1234" line in the issue description. -- The creation date is taken from the github issue, but can be overriden by adding a "Date: yyyy-mm-dd" line in the issue description. +- Please link to the proposal document by adding a "Documentation: " line + in the issue description. +- Please link to the spec PR (if any) by adding a "PRs: #1234" line in the + issue description. +- The creation date is taken from the github issue, but can be overriden by + adding a "Date: yyyy-mm-dd" line in the issue description. - Updated Date is taken from github. -- Author is the creator of the github issue, but can be overriden by adding a "Author: @username" line in the body of the issue description. Please make sure @username is a github user (include the @!) -- A shepherd can be assigned by adding a "Shepherd: @username" line in the issue description. Again, make sure this is a real Github user. +- Author is the creator of the github issue, but can be overriden by adding a + "Author: @username" line in the body of the issue description. Please make + sure @username is a github user (include the @!) +- A shepherd can be assigned by adding a "Shepherd: @username" line in the + issue description. Again, make sure this is a real Github user. From 84524df7830faadca24d9d8462ce03216467afc8 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 15 May 2018 17:38:33 +0100 Subject: [PATCH 042/308] spell out requirements for doc editing --- specification/proposals_intro.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index 97b1ea22..dae5ac99 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -9,8 +9,9 @@ The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: - produce a publicly-accessible proposal describing your change: - Please use Google Docs, or an equivalent system capable of collaborative - editing, with versioned history and threaded comments. Please ensure the - document is world-commentable or -editable. + editing, with versioned history, suggestions ('track changes'), threaded + comments, and good mobile support. Please ensure the document is + world-commentable or -editable. - We do not use Github issues (or Etherpad) for the design process of the proposal, as the document review/commenting capabilities aren't good enough. From 55b512c9683a439b9d6d3175cd8c92e8356ee077 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Mon, 14 May 2018 16:03:38 +0200 Subject: [PATCH 043/308] add delete_devices endpoint fix #1041 --- api/client-server/device_management.yaml | 47 ++++++++++++++++++++++++ changelogs/client_server.rst | 2 + 2 files changed, 49 insertions(+) diff --git a/api/client-server/device_management.yaml b/api/client-server/device_management.yaml index 030ed831..75ee9e44 100644 --- a/api/client-server/device_management.yaml +++ b/api/client-server/device_management.yaml @@ -177,3 +177,50 @@ paths: "$ref": "definitions/auth_response.yaml" tags: - Device management + "/delete_devices": + post: + summary: Bulk deletion of devices + description: |- + This API endpoint uses the `User-Interactive Authentication API`_. + + Deletes the given devices, and invalidates any access token associated with them. + operationId: deleteDevices + security: + - accessToken: [] + parameters: + - in: body + name: body + schema: + type: object + properties: + devices: + type: array + description: The list of device IDs to delete. + items: + type: string + description: A list of device IDs. + example: ["QBUAZIFURK", "AUIECTSRND"] + auth: + description: |- + Additional authentication information for the + user-interactive authentication API. + "$ref": "definitions/auth_data.yaml" + required: + - devices + responses: + 200: + description: |- + The devices were successfully removed, or had been removed + previously. + schema: + type: object + examples: + application/json: { + } + 401: + description: |- + The homeserver requires additional authentication information. + schema: + "$ref": "definitions/auth_response.yaml" + tags: + - Device management diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index c29f38c7..a2a929b0 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -9,6 +9,8 @@ Unreleased changes (`#1096 `_). - ``GET /rooms/{roomId}/event/{eventId}`` (`#1110 `_). + - ``POST /delete_devices`` + (`#1239 `_). - Sticker messages: - Add sticker message event definition. From 4d3c4225b21e8e3b4d4d883b45570515a6776d19 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Wed, 16 May 2018 11:12:57 +0100 Subject: [PATCH 044/308] include proposal template link --- specification/proposals_intro.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index dae5ac99..47393fa6 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -24,6 +24,9 @@ The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: - Proposal documents are intended to be as lightweight and flexible as the author desires; there is no formal template; the intention is to iterate as quickly as possible to get to a good design. + - A `template with suggested headers + `_ + is available. - make a new issue at https://github.com/matrix-org/matrix-doc/issues (or modify an existing one), whose description should list the metadata as per From cc54a4f8cb467af66c35ee3eb0d880329ff0b715 Mon Sep 17 00:00:00 2001 From: Florian Jacob Date: Thu, 17 May 2018 16:43:01 +0200 Subject: [PATCH 045/308] /rooms/{roomId}/members: specify access_token requirement as the behaviour of which members the users see is user-specific and therefore requires authentication. Signed-off-by: Florian Jacob --- api/client-server/rooms.yaml | 2 ++ changelogs/client_server.rst | 2 ++ 2 files changed, 4 insertions(+) diff --git a/api/client-server/rooms.yaml b/api/client-server/rooms.yaml index 88c2b9d9..cc1f2bf7 100644 --- a/api/client-server/rooms.yaml +++ b/api/client-server/rooms.yaml @@ -288,6 +288,8 @@ paths: description: The room to get the member events for. required: true x-example: "!636q39766251:example.com" + security: + - accessToken: [] responses: 200: description: |- diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index a2a929b0..3c4fbffa 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -35,6 +35,8 @@ Unreleased changes (`#1139 `_). - Clarify that ``/account/whoami`` should consider application services (`#1152 `_). + - Mark ``GET /rooms/{roomId}/members`` as requiring authentication + (`#1245 `_). - Changes to the API which will be backwards-compatible for clients: From 8440179ecfc5ce9d052fc3c8cfbca6256da0e4ad Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Thu, 17 May 2018 18:21:39 +0100 Subject: [PATCH 046/308] clarify shepherds and clarify 'greater benefit' as per https://github.com/matrix-org/matrix-doc/pull/1240#discussion_r188459957 --- specification/proposals_intro.rst | 41 ++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index 47393fa6..ce577337 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -45,9 +45,14 @@ The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: trying to run matrix apps (clients & servers); #matrix-architecture:matrix.org is for cross-cutting discussion of Matrix's architectural design. - - Iterating on the proposal and gathering consensus can sometimes be - time-consuming; an impartial 'shepherd' can be assigned to help guide the - proposal through this process. + - The point of the spec proposal process is to be collaborative rather than + competitive, and to try to solve the problem in question with the optimal + set of trade-offs. Ideally the author would neutrally gather the various + viewpoints and get consensus, but this can sometimes be time-consuming (or + the author may be biased), in which case an impartial 'shepherd' can be + assigned to help guide the proposal through this process. A shepherd is + typically a neutral party from the core team or an experienced member of + the community. - Once the proposal has sufficient consensus and passed review, you **must** show an implementation to prove that it works well in practice, before a @@ -69,6 +74,36 @@ rather than benefiting or privileging any single player or subset of players to act as a neutral custodian for Matrix on behalf of the whole ecosystem, just as it has since Matrix's inception in May 2014. +For clarity: the Matrix ecosystem is anyone who uses the Matrix protocol. That +includes client users, server admins, client developers, bot developers, +bridge and AS developers, users and admins who are indirectly using Matrix via +3rd party networks which happen to be bridged, server developers, room +moderators and admins, companies/projects building products or services on +Matrix, spec contributors, translators, and the core team who created it in +the first place. + +"Greater benefit" could include maximising: + + * the number of end-users reachable on the open Matrix network. + * the number of regular users on the Matrix network (e.g. 30-day retained + federated users) + * the number of online servers in the open federation. + * the number of developers building on Matrix. + * the number of independent implementations which use Matrix + * the quality and utility of the Matrix spec. + +The guiding principles of the overall project are being worked on as part of +the upcoming governance proposal, but could be something like: + + * Supporting the whole long-term ecosystem rather than individual stakeholder gain + * Openness rather than proprietariness + * Collaboration rather than competition + * Accessibility rather than elitism + * Transparency rather than stealth + * Empathy rather than contrariness + * Pragmatism rather than perfection + * Proof rather than conjecture + The above directions are intended to be simple and pragmatic rather than exhaustive, and aim to provide guidelines until we have a formal spec governance process in place that covers the whole Matrix community. In order From 38adc7df06743af333fd24aa04408e13aad9eff4 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Thu, 17 May 2018 18:25:13 +0100 Subject: [PATCH 047/308] latest generated proposals --- specification/proposals.rst | 618 +++++++++++++++++++++++++++++- specification/proposals_intro.rst | 30 +- 2 files changed, 629 insertions(+), 19 deletions(-) diff --git a/specification/proposals.rst b/specification/proposals.rst index 5a1f02c0..da9d277f 100644 --- a/specification/proposals.rst +++ b/specification/proposals.rst @@ -1,5 +1,615 @@ -.. contents:: Table of Contents -.. sectnum:: +Tables of Tracked Proposals +--------------------------- -The Proposals List ------------------- +proposal-wip +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documenation + - Author + - Shepherd + - PRs + * - `MSC455 `_ + - Do we want to specify a matrix:// URI scheme for rooms? (SPEC-5) + - 2014-09-15 + - 2018-05-15 + - `455-1 `_ + - `@KitsuneRal`_ + - None + - + * - `MSC586 `_ + - Federation API for canonicalising MXIDs + - 2014-10-27 + - 2018-05-15 + - `586-1 `_ + - `@ara4n`_ + - None + - + * - `MSC489 `_ + - Extensible Profiles. (SPEC-93) + - 2015-01-19 + - 2018-05-15 + - `489-1 `_ + - `@erikjohnston`_ + - None + - + * - `MSC1196 `_ + - Matrix Escape Hatch (Versioned Rooms) + - 2015-10-22 + - 2018-05-15 + - `1196-1 `_ + - `@leonerd`_ + - None + - + * - `MSC1238 `_ + - Push to Talk + - 2016-04-13 + - 2018-05-15 + - `1238-1 `_ + - `@aviraldg`_ + - None + - `PR#310`_ + * - `MSC1198 `_ + - Threading API + - 2016-05-23 + - 2018-05-15 + - `1198-1 `_ + - `@Half-Shot`_ + - None + - + * - `MSC1207 `_ + - Publishing Room Lists for 3rd party networks + - 2016-10-21 + - 2018-05-15 + - `1207-1 `_ + - `@erikjohnston`_ + - None + - + * - `MSC441 `_ + - Support for Reactions / Aggregations + - 2016-12-25 + - 2018-05-15 + - `441-1 `_ + - `@pik`_ + - `@ara4n`_ + - + * - `MSC1237 `_ + - Improving m.location with GeoJSON and accuracy + - 2017-05-19 + - 2018-05-15 + - `1237-1 `_ + - `@Half-Shot`_ + - None + - `PR#919`_ + * - `MSC971 `_ + - Add groups stuff to spec + - 2017-08-10 + - 2018-05-15 + - `971-1 `_, `971-2 `_, `971-3 `_ + - `@erikjohnston`_ + - None + - + * - `MSC1215 `_ + - Groups as Rooms + - 2017-10-17 + - 2018-05-15 + - `1215-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1217 `_ + - Visibility of groups to group members and their non-members + - 2017-10-30 + - 2018-05-15 + - `1217-1 `_ + - `@lampholder`_ + - None + - + * - `MSC1218 `_ + - Bridging group membership with 3rd party group sources + - 2017-11-15 + - 2018-05-15 + - `1218-1 `_ + - `@lukebarnard1`_ + - None + - + * - `MSC1219 `_ + - Proposal for storing megolm keys serverside + - 2017-11-23 + - 2018-05-15 + - `1219-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1221 `_ + - Improving Presence + - 2017-12-26 + - 2018-05-15 + - `1221-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1222 `_ + - Pushing updates about Groups (Communities) to clients + - 2018-01-02 + - 2018-05-15 + - `1222-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1225 `_ + - Extensible event types & fallback in Matrix + - 2018-02-18 + - 2018-05-15 + - `1225-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1227 `_ + - Proposal for lazy-loading room members to improve initial sync speed and client RAM usage + - 2018-03-05 + - 2018-05-15 + - `1227-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1206 `_ + - Proposal for improved bot support + - 2018-03-14 + - 2018-05-15 + - `1206-1 `_ + - `@turt2live`_ + - None + - + * - `MSC1228 `_ + - Removing MXIDs from events + - 2018-04-19 + - 2018-05-15 + - `1228-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1231 `_ + - Handling Consent for Privacy Policy + - 2018-05-02 + - 2018-05-15 + - `1231-1 `_ + - `@neilisfragile`_ + - None + - + + + +proposal-ready-for-review +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documenation + - Author + - Shepherd + - PRs + * - `MSC1183 `_ + - Document key-share requests + - 2018-04-30 + - 2018-05-15 + - `1183-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1232 `_ + - Media limits API + - 2018-05-04 + - 2018-05-15 + - `1232-1 `_ + - `@Half-Shot`_ + - None + - `PR#1189`_ + + + +proposal-in-review +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documenation + - Author + - Shepherd + - PRs + * - `MSC433 `_ + - Support for discovering API endpoints via .well-known URIs (SPEC-121) + - 2015-03-08 + - 2018-05-15 + - `433-1 `_, `433-2 `_ + - `@maxidor`_, `others`_ + - `@uhoreg`_ + - + * - `MSC1194 `_ + - A way for HSes to remove bindings from ISes + - 2018-05-09 + - 2018-05-15 + - `1194-1 `_ + - `@dbkr`_ + - None + - + + + +proposal-passed-review +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documenation + - Author + - Shepherd + - PRs + * - `MSC1226 `_ + - State Reset mitigation proposal + - 2018-02-20 + - 2018-05-15 + - `1226-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1229 `_ + - Mitigating abuse of the event depth parameter over federation + - 2018-04-30 + - 2018-05-15 + - `1229-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1233 `_ + - A proposal for organising spec proposals + - 2018-05-10 + - 2018-05-15 + - `1233-1 `_ + - `@ara4n`_ + - None + - `PR#1240`_ + * - `MSC1234 `_ + - Rich Replies format + - 2018-05-12 + - 2018-05-17 + - `1234-1 `_ + - `@t3chguy`_ + - None + - + * - `MSC1236 `_ + - Matrix Widget API v2 + - 2018-05-13 + - 2018-05-15 + - `1236-1 `_ + - `@rxl881`_ + - None + - + + + +spec-pr-missing +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documenation + - Author + - Shepherd + - PRs + * - `MSC1197 `_ + - Ignoring Users + - 2016-05-03 + - 2018-05-15 + - `1197-1 `_ + - `@erikjohnston`_ + - None + - + * - `MSC1201 `_ + - Device Management API + - 2016-07-14 + - 2018-05-15 + - `1201-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1203 `_ + - 3rd Party Entity lookup API + - 2016-07-21 + - 2018-05-15 + - `1203-1 `_ + - `@leonerd`_ + - None + - + * - `MSC1205 `_ + - Proposal for multi-device deletion API + - 2016-10-12 + - 2018-05-15 + - `1205-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1208 `_ + - Encrypted attachment format + - 2016-10-26 + - 2018-05-15 + - `1208-1 `_ + - `@NegativeMjark`_ + - None + - + * - `MSC739 `_ + - Reporting inappropriate content in Matrix + - 2016-11-21 + - 2018-05-15 + - `739-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1211 `_ + - Megolm session export format + - 2017-01-03 + - 2018-05-15 + - `1211-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1212 `_ + - Device List Update Stream + - 2017-01-18 + - 2018-05-15 + - `1212-1 `_ + - `@richvdh`_ + - None + - + * - `MSC829 `_ + - Need to spec msisdn login API + - 2017-03-08 + - 2018-05-15 + - `829-1 `_ + - `@dbkr`_ + - None + - + * - `MSC855 `_ + - spec m.login.msisdn UI auth type + - 2017-03-24 + - 2018-05-15 + - `855-1 `_ + - `@dbkr`_ + - None + - + * - `MSC910 `_ + - Add new Read Marker API to docs + - 2017-05-08 + - 2018-05-15 + - + - `@lukebarnard1`_ + - None + - + * - `MSC1067 `_ + - Spec @mentions + - 2017-07-14 + - 2018-05-15 + - `1067-1 `_ + - `@lukebarnard1`_ + - None + - + * - `MSC1214 `_ + - Related Groups (i.e. flair) + - 2017-10-03 + - 2018-05-15 + - `1214-1 `_ + - `@lukebarnard1`_ + - None + - + * - `MSC1216 `_ + - @room notification proposal + - 2017-10-23 + - 2018-05-15 + - `1216-1 `_ + - `@dbkr`_ + - None + - `PR#1176`_ + * - `MSC1230 `_ + - Temporary mitigation for depth parameter abuse + - 2018-05-01 + - 2018-05-15 + - `1230-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1234 `_ + - Rich Replies format + - 2018-05-12 + - 2018-05-17 + - `1234-1 `_ + - `@t3chguy`_ + - None + - + + + +merged +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documenation + - Author + - Shepherd + - PRs + * - `MSC1199 `_ + - Notifications API + - 2016-05-23 + - 2018-05-15 + - `1199-1 `_ + - `@dbkr`_ + - None + - + * - `MSC1200 `_ + - Configuration of E2E encryption in a room + - 2016-06-16 + - 2018-05-15 + - `1200-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1204 `_ + - Access Token Semantics (refresh and macaroons) - aka Auth Sept 2016 Edition + - 2016-09-29 + - 2018-05-15 + - `1204-1 `_ + - `@richvdh`_ + - None + - + + + +abandoned +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documenation + - Author + - Shepherd + - PRs + * - `MSC531 `_ + - Homeservers as OAuth authorization endpoints (resource owners) (SPEC-206) + - 2015-07-25 + - 2018-05-15 + - `531-1 `_ + - `@Kegsay`_ + - None + - + * - `MSC1202 `_ + - Profile Personae + - 2016-07-15 + - 2018-05-15 + - `1202-1 `_ + - `@erikjohnston`_ + - None + - + * - `MSC1209 `_ + - Server Side Profile API + - 2016-11-01 + - 2018-05-15 + - `1209-1 `_ + - `@erikjohnston`_ + - None + - + * - `MSC1213 `_ + - Set defaults for m.federate + - 2018-05-10 + - 2018-05-15 + - `1213-1 `_ + - `@psaavedra`_ + - None + - + + + +obsolete +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documenation + - Author + - Shepherd + - PRs + * - `MSC1235 `_ + - Proposal for Calendar Events + - 2018-02-06 + - 2018-05-15 + - `1235-1 `_ + - `@Half-Shot`_ + - None + - + + + + + +.. _@t3chguy: https://github.com/t3chguy +.. _@KitsuneRal: https://github.com/KitsuneRal +.. _@pik: https://github.com/pik +.. _@leonerd: https://github.com/leonerd +.. _@turt2live: https://github.com/turt2live +.. _@erikjohnston: https://github.com/erikjohnston +.. _@neilisfragile: https://github.com/neilisfragile +.. _@psaavedra: https://github.com/psaavedra +.. _@richvdh: https://github.com/richvdh +.. _@NegativeMjark: https://github.com/NegativeMjark +.. _@ara4n: https://github.com/ara4n +.. _@lukebarnard1: https://github.com/lukebarnard1 +.. _@lampholder: https://github.com/lampholder +.. _@dbkr: https://github.com/dbkr +.. _@maxidor: https://github.com/maxidor +.. _others: https://github.com/thers +.. _@rxl881: https://github.com/rxl881 +.. _@uhoreg: https://github.com/uhoreg +.. _@Kegsay: https://github.com/Kegsay +.. _@Half-Shot: https://github.com/Half-Shot +.. _@aviraldg: https://github.com/aviraldg +.. _PR#310: https://github.com/matrix-org/matrix-doc/pull/310 +.. _PR#1240: https://github.com/matrix-org/matrix-doc/pull/1240 +.. _PR#1189: https://github.com/matrix-org/matrix-doc/pull/1189 +.. _PR#919: https://github.com/matrix-org/matrix-doc/pull/919 +.. _PR#1176: https://github.com/matrix-org/matrix-doc/pull/1176 \ No newline at end of file diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index ce577337..fecfc2f9 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -84,25 +84,25 @@ the first place. "Greater benefit" could include maximising: - * the number of end-users reachable on the open Matrix network. - * the number of regular users on the Matrix network (e.g. 30-day retained - federated users) - * the number of online servers in the open federation. - * the number of developers building on Matrix. - * the number of independent implementations which use Matrix - * the quality and utility of the Matrix spec. +* the number of end-users reachable on the open Matrix network. +* the number of regular users on the Matrix network (e.g. 30-day retained + federated users) +* the number of online servers in the open federation. +* the number of developers building on Matrix. +* the number of independent implementations which use Matrix +* the quality and utility of the Matrix spec. The guiding principles of the overall project are being worked on as part of the upcoming governance proposal, but could be something like: - * Supporting the whole long-term ecosystem rather than individual stakeholder gain - * Openness rather than proprietariness - * Collaboration rather than competition - * Accessibility rather than elitism - * Transparency rather than stealth - * Empathy rather than contrariness - * Pragmatism rather than perfection - * Proof rather than conjecture +* Supporting the whole long-term ecosystem rather than individual stakeholder gain +* Openness rather than proprietariness +* Collaboration rather than competition +* Accessibility rather than elitism +* Transparency rather than stealth +* Empathy rather than contrariness +* Pragmatism rather than perfection +* Proof rather than conjecture The above directions are intended to be simple and pragmatic rather than exhaustive, and aim to provide guidelines until we have a formal spec From 9873a2a0ac40dfcdce4bac71f61d849f2172570e Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Thu, 17 May 2018 18:33:12 +0100 Subject: [PATCH 048/308] link from the spec intro page --- specification/intro.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/specification/intro.rst b/specification/intro.rst index 064f9d17..ad545248 100644 --- a/specification/intro.rst +++ b/specification/intro.rst @@ -96,6 +96,13 @@ instant messages, VoIP call setups, or any other objects that need to be reliably and persistently pushed from A to B in an interoperable and federated manner. + +Spec Change Proposals +~~~~~~~~~~~~~~~~~~~~~ +To propose a change to the Matrix Spec, see the explanations at `Proposals +for Spec Changes to Matrix `_. + + Architecture ------------ From 663e1a5ebec7d95856cbc214aaa26355e68e880b Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Fri, 18 May 2018 10:17:54 +0100 Subject: [PATCH 049/308] Update some text --- scripts/proposals.py | 2 +- specification/proposals.rst | 54 +++++++++++-------------------- specification/proposals_intro.rst | 4 +-- 3 files changed, 22 insertions(+), 38 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index 8b2107ac..a84804a7 100644 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -55,7 +55,7 @@ for label in labels: text_file.write(" - Proposal Title\n") text_file.write(" - Creation Date\n") text_file.write(" - Update Date\n") - text_file.write(" - Documenation\n") + text_file.write(" - Documentation\n") text_file.write(" - Author\n") text_file.write(" - Shepherd\n") text_file.write(" - PRs\n") diff --git a/specification/proposals.rst b/specification/proposals.rst index da9d277f..c8d58764 100644 --- a/specification/proposals.rst +++ b/specification/proposals.rst @@ -13,7 +13,7 @@ proposal-wip - Proposal Title - Creation Date - Update Date - - Documenation + - Documentation - Author - Shepherd - PRs @@ -200,7 +200,7 @@ proposal-ready-for-review - Proposal Title - Creation Date - Update Date - - Documenation + - Documentation - Author - Shepherd - PRs @@ -235,7 +235,7 @@ proposal-in-review - Proposal Title - Creation Date - Update Date - - Documenation + - Documentation - Author - Shepherd - PRs @@ -270,7 +270,7 @@ proposal-passed-review - Proposal Title - Creation Date - Update Date - - Documenation + - Documentation - Author - Shepherd - PRs @@ -290,22 +290,6 @@ proposal-passed-review - `@ara4n`_ - None - - * - `MSC1233 `_ - - A proposal for organising spec proposals - - 2018-05-10 - - 2018-05-15 - - `1233-1 `_ - - `@ara4n`_ - - None - - `PR#1240`_ - * - `MSC1234 `_ - - Rich Replies format - - 2018-05-12 - - 2018-05-17 - - `1234-1 `_ - - `@t3chguy`_ - - None - - * - `MSC1236 `_ - Matrix Widget API v2 - 2018-05-13 @@ -329,18 +313,10 @@ spec-pr-missing - Proposal Title - Creation Date - Update Date - - Documenation + - Documentation - Author - Shepherd - PRs - * - `MSC1197 `_ - - Ignoring Users - - 2016-05-03 - - 2018-05-15 - - `1197-1 `_ - - `@erikjohnston`_ - - None - - * - `MSC1201 `_ - Device Management API - 2016-07-14 @@ -456,7 +432,7 @@ spec-pr-missing * - `MSC1234 `_ - Rich Replies format - 2018-05-12 - - 2018-05-17 + - 2018-05-18 - `1234-1 `_ - `@t3chguy`_ - None @@ -476,7 +452,7 @@ merged - Proposal Title - Creation Date - Update Date - - Documenation + - Documentation - Author - Shepherd - PRs @@ -504,6 +480,14 @@ merged - `@richvdh`_ - None - + * - `MSC1233 `_ + - A proposal for organising spec proposals + - 2018-05-10 + - 2018-05-18 + - `1233-1 `_ + - `@ara4n`_ + - None + - `PR#1240`_ @@ -519,7 +503,7 @@ abandoned - Proposal Title - Creation Date - Update Date - - Documenation + - Documentation - Author - Shepherd - PRs @@ -549,8 +533,8 @@ abandoned - * - `MSC1213 `_ - Set defaults for m.federate - - 2018-05-10 - - 2018-05-15 + - 2017-04-10 + - 2018-05-18 - `1213-1 `_ - `@psaavedra`_ - None @@ -570,7 +554,7 @@ obsolete - Proposal Title - Creation Date - Update Date - - Documenation + - Documentation - Author - Shepherd - PRs diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index fecfc2f9..fdc34a29 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -6,7 +6,7 @@ Proposals for Spec Changes to Matrix The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: -- produce a publicly-accessible proposal describing your change: +- Produce a publicly-accessible proposal describing your change: - Please use Google Docs, or an equivalent system capable of collaborative editing, with versioned history, suggestions ('track changes'), threaded @@ -28,7 +28,7 @@ The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: `_ is available. -- make a new issue at https://github.com/matrix-org/matrix-doc/issues (or +- Make a new issue at https://github.com/matrix-org/matrix-doc/issues (or modify an existing one), whose description should list the metadata as per below. - Gather feedback as widely as possible from the community and core team on From 2a6895e7591222517609b727d47097431851567c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 18 May 2018 17:06:22 -0700 Subject: [PATCH 050/308] Fix typo. --- specification/modules/history_visibility.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/history_visibility.rst b/specification/modules/history_visibility.rst index 476ea889..84435edb 100644 --- a/specification/modules/history_visibility.rst +++ b/specification/modules/history_visibility.rst @@ -82,7 +82,7 @@ For ``m.room.history_visibility`` events themselves, the user should be allowed to see the event if the ``history_visibility`` before *or* after the event would allow them to see it. (For example, a user should be able to see ``m.room.history_visibility`` events which change the ``history_visibility`` -from ``world_readable`` to ``joined`` *or* from ``joineded`` to +from ``world_readable`` to ``joined`` *or* from ``joined`` to ``world_readable``, even if that user was not a member of the room.) Likewise, for the user's own ``m.room.member`` events, the user should be From 0b776b0e9cfc9a609b4e20a05b74f857296af0fa Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Mon, 21 May 2018 11:01:03 +0100 Subject: [PATCH 051/308] link to proposals page from index --- specification/index.rst | 3 +++ specification/proposals.rst | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/specification/index.rst b/specification/index.rst index 5739ce09..488f07d7 100644 --- a/specification/index.rst +++ b/specification/index.rst @@ -27,6 +27,9 @@ Voice over IP (VoIP) signalling, Internet of Things (IoT) communication, and bri together existing communication silos - providing the basis of a new open real-time communication ecosystem. +To propose a change to the Matrix Spec, see the explanations at `Proposals +for Spec Changes to Matrix `_. + Matrix APIs ----------- diff --git a/specification/proposals.rst b/specification/proposals.rst index c8d58764..d36b93dc 100644 --- a/specification/proposals.rst +++ b/specification/proposals.rst @@ -92,7 +92,7 @@ proposal-wip * - `MSC971 `_ - Add groups stuff to spec - 2017-08-10 - - 2018-05-15 + - 2018-05-20 - `971-1 `_, `971-2 `_, `971-3 `_ - `@erikjohnston`_ - None From 8d550d21666e55b430cbc3739ee0d2bbc84aa6e5 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Wed, 23 May 2018 09:39:18 +0100 Subject: [PATCH 052/308] force re-render --- specification/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specification/index.rst b/specification/index.rst index 488f07d7..f9a3fef3 100644 --- a/specification/index.rst +++ b/specification/index.rst @@ -27,8 +27,8 @@ Voice over IP (VoIP) signalling, Internet of Things (IoT) communication, and bri together existing communication silos - providing the basis of a new open real-time communication ecosystem. -To propose a change to the Matrix Spec, see the explanations at `Proposals -for Spec Changes to Matrix `_. +To propose a change to the Matrix Spec, see the explanations at +`Proposals for Spec Changes to Matrix `_. Matrix APIs ----------- From 434e2949f919cd909e8ac93c01f4b015c7d19a1f Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Wed, 23 May 2018 17:00:26 +0100 Subject: [PATCH 053/308] include #1256 --- specification/proposals.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/specification/proposals.rst b/specification/proposals.rst index d36b93dc..7872f6bd 100644 --- a/specification/proposals.rst +++ b/specification/proposals.rst @@ -185,6 +185,14 @@ proposal-wip - `@neilisfragile`_ - None - + * - `MSC1256 `_ + - Custom emoji and sticker packs in matrix + - 2018-05-23 + - 2018-05-23 + - `1256-1 `_ + - `@turt2live`_ + - None + - From c3062b7db6360511bf9643f7f080f4f2c99557a4 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Thu, 24 May 2018 11:36:56 +0100 Subject: [PATCH 054/308] Include Proposals Page Title, update MSC1256 --- specification/proposals.rst | 18 +++++++++--------- specification/proposals_intro.rst | 2 ++ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/specification/proposals.rst b/specification/proposals.rst index 7872f6bd..80d1ce0f 100644 --- a/specification/proposals.rst +++ b/specification/proposals.rst @@ -132,7 +132,7 @@ proposal-wip * - `MSC1221 `_ - Improving Presence - 2017-12-26 - - 2018-05-15 + - 2018-05-24 - `1221-1 `_ - `@ara4n`_ - None @@ -185,14 +185,6 @@ proposal-wip - `@neilisfragile`_ - None - - * - `MSC1256 `_ - - Custom emoji and sticker packs in matrix - - 2018-05-23 - - 2018-05-23 - - `1256-1 `_ - - `@turt2live`_ - - None - - @@ -228,6 +220,14 @@ proposal-ready-for-review - `@Half-Shot`_ - None - `PR#1189`_ + * - `MSC1256 `_ + - Custom emoji and sticker packs in matrix + - 2018-05-23 + - 2018-05-24 + - `1256-1 `_ + - `@turt2live`_ + - None + - diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index fdc34a29..cfa59b8f 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -1,3 +1,5 @@ +.. title:: Proposals for Spec Changes to Matrix + .. contents:: Table of Contents .. sectnum:: From 4278cd56a40f4d81037dbba033e7a19af1c2f42c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 27 May 2018 11:45:36 -0600 Subject: [PATCH 055/308] Document /logout/all Fixes https://github.com/matrix-org/matrix-doc/issues/700 Signed-off-by: Travis Ralston --- api/client-server/logout.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/api/client-server/logout.yaml b/api/client-server/logout.yaml index 3a2f71e9..2dfd6d97 100644 --- a/api/client-server/logout.yaml +++ b/api/client-server/logout.yaml @@ -44,3 +44,26 @@ paths: properties: {} tags: - Session management + "/logout/all": + post: + summary: Invalidates all access tokens for a user + description: |- + Invalidates all access tokens for a user, so that they can no longer be used for + authorization. This includes the access token that made this request. + + This endpoint does not require UI authorization because UI authorization is + designed to protect against attacks where the someone gets hold of a single access + token then takes over the account. This endpoint invalidates all access tokens for + the user, including the token used in the request, and therefore the attacker is + unable to take over the account in this way. + operationId: logout_all + security: + - accessToken: [] + responses: + 200: + description: The user's access tokens were succesfully invalidated. + schema: + type: object + properties: {} + tags: + - Session management From 59ea10b84282dba6dd8d1af0412716d218bd6d04 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 27 May 2018 11:47:28 -0600 Subject: [PATCH 056/308] Add changelog entry for /logout/all Signed-off-by: Travis Ralston --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 3c4fbffa..9833662e 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -48,6 +48,8 @@ Unreleased changes (`#1142 `_). - Add the ``/register/available`` endpoint for username availability (`#1151 `_). + - Document ``/logout/all`` endpoint + (`#1263 `_). r0.3.0 ====== From 9037f506acf528bb73886f058451b314912419b4 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 27 May 2018 12:19:30 -0600 Subject: [PATCH 057/308] Document how to report content Fixes: https://github.com/matrix-org/matrix-doc/issues/739 Signed-off-by: Travis Ralston --- api/client-server/report_content.yaml | 78 ++++++++++++++++++++++++ specification/modules/report_content.rst | 35 +++++++++++ specification/targets.yaml | 1 + 3 files changed, 114 insertions(+) create mode 100644 api/client-server/report_content.yaml create mode 100644 specification/modules/report_content.rst diff --git a/api/client-server/report_content.yaml b/api/client-server/report_content.yaml new file mode 100644 index 00000000..f702151d --- /dev/null +++ b/api/client-server/report_content.yaml @@ -0,0 +1,78 @@ +# Copyright 2018 Travis Ralston +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +swagger: '2.0' +info: + title: "Matrix Client-Server Report Content API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/%CLIENT_MAJOR_VERSION% +consumes: + - application/json +produces: + - application/json +securityDefinitions: + $ref: definitions/security.yaml +paths: + "/rooms/{roomId}/report/{eventId}": + post: + summary: Reports an event as inappropriate. + description: |- + Reports an event as inappropriate to the server, which may then notify + the appropriate people. + operationId: reportContent + parameters: + - in: path + type: string + name: roomId + description: The room in which the event being reported is located. + required: true + x-example: "!637q39766251:example.com" + - in: path + type: string + name: eventId + description: The event to report. + required: true + x-example: "$something:domain.com" + - in: body + name: body + schema: + type: object + example: { + "score": -100, + "reason": "this makes me sad" + } + required: ['score', 'reason'] + properties: + score: + type: integer + description: |- + The score to rate this content as where -100 is most offensive + and 0 is inoffensive. + reason: + type: string + description: The reason the content is being reported. May be blank. + security: + - accessToken: [] + responses: + 200: + description: The event has been reported successfully. + schema: + type: object + examples: + application/json: {} + tags: + - Reporting content diff --git a/specification/modules/report_content.rst b/specification/modules/report_content.rst new file mode 100644 index 00000000..5eca69cd --- /dev/null +++ b/specification/modules/report_content.rst @@ -0,0 +1,35 @@ +.. Copyright 2018 Travis Ralston +.. +.. Licensed under the Apache License, Version 2.0 (the "License"); +.. you may not use this file except in compliance with the License. +.. You may obtain a copy of the License at +.. +.. http://www.apache.org/licenses/LICENSE-2.0 +.. +.. Unless required by applicable law or agreed to in writing, software +.. distributed under the License is distributed on an "AS IS" BASIS, +.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +.. See the License for the specific language governing permissions and +.. limitations under the License. + +Reporting Content +================= + +.. _module:report_content: + +Users may encounter content which they find inappropriate and should be able +to report it to the server administrators or room moderators for review. This +module defines a way for users to report content. + +Content is reported based upon a negative score, where -100 is "most offensive" +and 0 is "inoffensive". + +Client behaviour +---------------- +{{report_content_cs_http_api}} + +Server behaviour +---------------- +Servers are free to handle the reported content however they desire. This may +be a dedicated room to alert server administrators to the reported content or +some other mechanism for notifying the appropriate people. diff --git a/specification/targets.yaml b/specification/targets.yaml index 8ca5ce30..bc9d94c8 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -67,6 +67,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/dm.rst - modules/ignore_users.rst - modules/stickers.rst + - modules/report_content.rst title_styles: ["=", "-", "~", "+", "^", "`", "@", ":"] From 352f1b7776286b293b7d55fffed07162e3cf9d9a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 27 May 2018 12:23:46 -0600 Subject: [PATCH 058/308] Add report content API to changelog Signed-off-by: Travis Ralston --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 3c4fbffa..466cafbc 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -48,6 +48,8 @@ Unreleased changes (`#1142 `_). - Add the ``/register/available`` endpoint for username availability (`#1151 `_). + - Add report content API + (`#1264 `_). r0.3.0 ====== From 7cc15f331554e7e329d396275ba2c8f256d7a43f Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 27 May 2018 12:34:58 -0600 Subject: [PATCH 059/308] Document the allow_remote query param on the media repo Fixes https://github.com/matrix-org/matrix-doc/issues/837 Signed-off-by: Travis Ralston --- api/client-server/content-repo.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 03a25fd9..f76b2fb9 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -95,6 +95,15 @@ paths: required: true description: | The media ID from the ``mxc://`` URI (the path component) + - in: query + type: boolean + name: allow_remote + x-example: false + required: false + description: | + Indicates to the server that it should not attempt to fetch the media if it is deemed + remote. This is to prevent routing loops where the server contacts itself. Defaults to + true if not provided. responses: 200: description: "The content that was previously uploaded." @@ -140,6 +149,15 @@ paths: required: true description: | The filename to give in the Content-Disposition + - in: query + type: boolean + name: allow_remote + x-example: false + required: false + description: | + Indicates to the server that it should not attempt to fetch the media if it is deemed + remote. This is to prevent routing loops where the server contacts itself. Defaults to + true if not provided. responses: 200: description: "The content that was previously uploaded." @@ -198,6 +216,15 @@ paths: name: method x-example: "scale" description: The desired resizing method. + - in: query + type: boolean + name: allow_remote + x-example: false + required: false + description: | + Indicates to the server that it should not attempt to fetch the media if it is deemed + remote. This is to prevent routing loops where the server contacts itself. Defaults to + true if not provided. responses: 200: description: "A thumbnail of the requested content." From 8cbd80265085c7d0726b380525b08b1699d34af2 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 27 May 2018 12:36:50 -0600 Subject: [PATCH 060/308] Add allow_remote to the changelog Signed-off-by: Travis Ralston --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 3c4fbffa..422c3e2d 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -48,6 +48,8 @@ Unreleased changes (`#1142 `_). - Add the ``/register/available`` endpoint for username availability (`#1151 `_). + - Add ``allow_remote`` to the content repo to avoid routing loops + (`#1265 `_). r0.3.0 ====== From 34090b24c432ce4b94da59884e3e136db1aa8013 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Tue, 29 May 2018 14:19:03 +0100 Subject: [PATCH 061/308] prepare proposals.py for CI --- scripts/proposals.py | 4 ++-- specification/proposals.rst | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index a84804a7..33fa6416 100644 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -11,7 +11,7 @@ authors = set() prs = set() def getpage(url, page): - resp = requests.get(url + str(page)) + resp = requests.get(url + str(page), auth=('benparsons', '99fbc9df2d75a18b5ac7f24e326e4de548b06172')) for link in resp.links.values(): if link['rel'] == 'last': @@ -100,7 +100,7 @@ for label in labels: text_file.write(" - " + updated.strftime('%Y-%m-%d') + "\n") # list of document links (urls comma-separated) - maindoc = re.search('^Documentation: (.+?)\n', str(item['body'])) + maindoc = re.search('^Documentation: (.+?)$', str(item['body']), flags=re.MULTILINE) if maindoc is not None: maindoc = maindoc.group(1) doc_list_formatted = ["`" + str(item['number']) + "-" + str(i) + " <" + x.strip() + ">`_" for i, x in enumerate(maindoc.split(','),1)] diff --git a/specification/proposals.rst b/specification/proposals.rst index 80d1ce0f..a19d820f 100644 --- a/specification/proposals.rst +++ b/specification/proposals.rst @@ -140,7 +140,7 @@ proposal-wip * - `MSC1222 `_ - Pushing updates about Groups (Communities) to clients - 2018-01-02 - - 2018-05-15 + - 2018-05-24 - `1222-1 `_ - `@ara4n`_ - None @@ -156,7 +156,7 @@ proposal-wip * - `MSC1227 `_ - Proposal for lazy-loading room members to improve initial sync speed and client RAM usage - 2018-03-05 - - 2018-05-15 + - 2018-05-28 - `1227-1 `_ - `@ara4n`_ - None @@ -185,6 +185,14 @@ proposal-wip - `@neilisfragile`_ - None - + * - `MSC1267 `_ + - Interactive Key Verification + - 2018-05-28 + - 2018-05-28 + - `1267-1 `_ + - `@uhoreg`_ + - None + - From 7a82a5d4279f47560258b389433c7835bb919b67 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Tue, 29 May 2018 14:20:48 +0100 Subject: [PATCH 062/308] correct proposals.py --- scripts/proposals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index 33fa6416..5bbe3d08 100644 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -11,7 +11,7 @@ authors = set() prs = set() def getpage(url, page): - resp = requests.get(url + str(page), auth=('benparsons', '99fbc9df2d75a18b5ac7f24e326e4de548b06172')) + resp = requests.get(url + str(page)) for link in resp.links.values(): if link['rel'] == 'last': From 419bf1e52cd5dabc1428df7365ee77e726718190 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 30 May 2018 09:42:56 +0100 Subject: [PATCH 063/308] Fix continuserv instructions As per f04b17f1c90df655771aa53e71900329cccd1378 Also dedup continuserv readme --- README.rst | 10 +++++----- scripts/continuserv/README | 6 ------ scripts/continuserv/README.md | 3 +++ 3 files changed, 8 insertions(+), 11 deletions(-) delete mode 100644 scripts/continuserv/README create mode 100644 scripts/continuserv/README.md diff --git a/README.rst b/README.rst index 76f4ad55..6c87201c 100644 --- a/README.rst +++ b/README.rst @@ -68,10 +68,10 @@ Windows users ~~~~~~~~~~~~~ If you're on Windows Vista or higher, be sure that the "Symbolic Links" -option was selected when installing Git prior to cloning this repository. If -you're still seeing errors about files not being found it is likely because -the symlink at ``api/client-server/definitions/event-schemas`` looks like a -file. To correct the problem, open an Administrative/Elevated shell in your +option was selected when installing Git prior to cloning this repository. If +you're still seeing errors about files not being found it is likely because +the symlink at ``api/client-server/definitions/event-schemas`` looks like a +file. To correct the problem, open an Administrative/Elevated shell in your cloned matrix-doc directory and run the following:: cd api\client-server\definitions @@ -121,7 +121,7 @@ changes. It is written in Go, so you will need the ``go`` compiler installed on your computer. You will also need to install fsnotify by running:: - go get gopkg.in/fsnotify.v1 + go get gopkg.in/fsnotify/fsnotify.v1 Then, create a virtualenv as described above under `Preparation`_, and:: diff --git a/scripts/continuserv/README b/scripts/continuserv/README deleted file mode 100644 index edb9aef4..00000000 --- a/scripts/continuserv/README +++ /dev/null @@ -1,6 +0,0 @@ -continuserv proactively re-generates the spec on filesystem changes, and serves it over HTTP. - -To run it, you must install the `go` tool. You will also need to install fsnotify by running: - `go get gopkg.in/fsnotify/fsnotify.v1` -You can then run continuserv by running: - `go run main.go` diff --git a/scripts/continuserv/README.md b/scripts/continuserv/README.md new file mode 100644 index 00000000..40321bb6 --- /dev/null +++ b/scripts/continuserv/README.md @@ -0,0 +1,3 @@ +continuserv proactively re-generates the spec on filesystem changes, and serves +it over HTTP. For notes on using it, see [the main +readme](../../README.rst#continuserv). From cb00fc35abfb53429f0f495cde58d207c3be4bff Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Thu, 31 May 2018 15:54:23 +0100 Subject: [PATCH 064/308] git checkout master --- scripts/proposals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index 5bbe3d08..241b0708 100644 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -22,7 +22,7 @@ def getpage(url, page): def getbylabel(label): pagecount = 1 json = list() - urlbase = 'https://api.github.com/repos/matrix-org/matrix-doc/issues?state=open&labels=' + label + '&page=' + urlbase = 'https://api.github.com/repos/matrix-org/matrix-doc/issues?state=all&labels=' + label + '&page=' print(urlbase) json.extend(getpage(urlbase, 1)) for page in range(2, int(pagecount) + 1): From 7a44351a34f8b1fb7d08dfd498f162e2d4c3f83d Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Thu, 31 May 2018 16:13:31 +0100 Subject: [PATCH 065/308] update following rvdh spec blitz --- specification/proposals.rst | 121 +++++++++++++++++++++++++----------- 1 file changed, 85 insertions(+), 36 deletions(-) diff --git a/specification/proposals.rst b/specification/proposals.rst index a19d820f..636877ff 100644 --- a/specification/proposals.rst +++ b/specification/proposals.rst @@ -68,7 +68,7 @@ proposal-wip * - `MSC1207 `_ - Publishing Room Lists for 3rd party networks - 2016-10-21 - - 2018-05-15 + - 2018-05-31 - `1207-1 `_ - `@erikjohnston`_ - None @@ -212,14 +212,6 @@ proposal-ready-for-review - Author - Shepherd - PRs - * - `MSC1183 `_ - - Document key-share requests - - 2018-04-30 - - 2018-05-15 - - `1183-1 `_ - - `@richvdh`_ - - None - - * - `MSC1232 `_ - Media limits API - 2018-05-04 @@ -236,6 +228,14 @@ proposal-ready-for-review - `@turt2live`_ - None - + * - `MSC1270 `_ + - Synapse /_matrix/media/v1/resolve_url to API download and preview urls in the clients despite on CORS + - 2018-05-31 + - 2018-05-31 + - `1270-1 `_ + - `@oivoodoo`_ + - None + - @@ -333,6 +333,14 @@ spec-pr-missing - Author - Shepherd - PRs + * - `MSC1200 `_ + - Configuration of E2E encryption in a room + - 2016-06-16 + - 2018-05-31 + - `1200-1 `_ + - `@richvdh`_ + - None + - * - `MSC1201 `_ - Device Management API - 2016-07-14 @@ -429,14 +437,22 @@ spec-pr-missing - `@lukebarnard1`_ - None - - * - `MSC1216 `_ - - @room notification proposal + * - `MSC1033 `_ + - Doc @room notifications - 2017-10-23 - - 2018-05-15 - - `1216-1 `_ + - 2018-05-31 + - - `@dbkr`_ - None - - `PR#1176`_ + - + * - `MSC1183 `_ + - Document key-share requests + - 2018-04-30 + - 2018-05-31 + - `1183-1 `_ + - `@richvdh`_ + - None + - * - `MSC1230 `_ - Temporary mitigation for depth parameter abuse - 2018-05-01 @@ -472,6 +488,14 @@ merged - Author - Shepherd - PRs + * - `MSC1197 `_ + - Ignoring Users + - 2016-05-03 + - 2018-05-18 + - `1197-1 `_ + - `@erikjohnston`_ + - None + - `PR#1142`_ * - `MSC1199 `_ - Notifications API - 2016-05-23 @@ -480,14 +504,6 @@ merged - `@dbkr`_ - None - - * - `MSC1200 `_ - - Configuration of E2E encryption in a room - - 2016-06-16 - - 2018-05-15 - - `1200-1 `_ - - `@richvdh`_ - - None - - * - `MSC1204 `_ - Access Token Semantics (refresh and macaroons) - aka Auth Sept 2016 Edition - 2016-09-29 @@ -496,6 +512,14 @@ merged - `@richvdh`_ - None - + * - `MSC953 `_ + - Add /user_directory/search API + - 2017-05-31 + - 2018-05-10 + - `953-1 `_ + - `@erikjohnston`_ + - None + - * - `MSC1233 `_ - A proposal for organising spec proposals - 2018-05-10 @@ -574,6 +598,22 @@ obsolete - Author - Shepherd - PRs + * - `MSC1223 `_ + - Replies event format + - 2018-02-01 + - 2018-05-15 + - `1223-1 `_ + - `@t3chguy`_ + - None + - + * - `MSC1224 `_ + - Replies - next steps + - 2018-02-03 + - 2018-05-15 + - `1224-1 `_ + - `@t3chguy`_ + - None + - * - `MSC1235 `_ - Proposal for Calendar Events - 2018-02-06 @@ -582,34 +622,43 @@ obsolete - `@Half-Shot`_ - None - + * - `MSC1220 `_ + - Rich quoting proposal + - 2018-05-10 + - 2018-05-15 + - `1220-1 `_ + - `@t3chguy`_ + - None + - +.. _@rxl881: https://github.com/rxl881 +.. _@psaavedra: https://github.com/psaavedra +.. _@erikjohnston: https://github.com/erikjohnston .. _@t3chguy: https://github.com/t3chguy +.. _@Kegsay: https://github.com/Kegsay .. _@KitsuneRal: https://github.com/KitsuneRal -.. _@pik: https://github.com/pik .. _@leonerd: https://github.com/leonerd .. _@turt2live: https://github.com/turt2live -.. _@erikjohnston: https://github.com/erikjohnston -.. _@neilisfragile: https://github.com/neilisfragile -.. _@psaavedra: https://github.com/psaavedra -.. _@richvdh: https://github.com/richvdh -.. _@NegativeMjark: https://github.com/NegativeMjark .. _@ara4n: https://github.com/ara4n -.. _@lukebarnard1: https://github.com/lukebarnard1 -.. _@lampholder: https://github.com/lampholder -.. _@dbkr: https://github.com/dbkr .. _@maxidor: https://github.com/maxidor -.. _others: https://github.com/thers -.. _@rxl881: https://github.com/rxl881 .. _@uhoreg: https://github.com/uhoreg -.. _@Kegsay: https://github.com/Kegsay +.. _@pik: https://github.com/pik +.. _@neilisfragile: https://github.com/neilisfragile +.. _@lukebarnard1: https://github.com/lukebarnard1 +.. _others: https://github.com/thers .. _@Half-Shot: https://github.com/Half-Shot .. _@aviraldg: https://github.com/aviraldg +.. _@oivoodoo: https://github.com/oivoodoo +.. _@richvdh: https://github.com/richvdh +.. _@NegativeMjark: https://github.com/NegativeMjark +.. _@lampholder: https://github.com/lampholder +.. _@dbkr: https://github.com/dbkr .. _PR#310: https://github.com/matrix-org/matrix-doc/pull/310 +.. _PR#1142: https://github.com/matrix-org/matrix-doc/pull/1142 .. _PR#1240: https://github.com/matrix-org/matrix-doc/pull/1240 .. _PR#1189: https://github.com/matrix-org/matrix-doc/pull/1189 -.. _PR#919: https://github.com/matrix-org/matrix-doc/pull/919 -.. _PR#1176: https://github.com/matrix-org/matrix-doc/pull/1176 \ No newline at end of file +.. _PR#919: https://github.com/matrix-org/matrix-doc/pull/919 \ No newline at end of file From 42fba1a87ec5886a2cba9ee60359347ca2647234 Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Sun, 3 Jun 2018 12:08:19 +0900 Subject: [PATCH 066/308] Add 'default' to formally enforce the description A quick follow up on #1265. Signed-off-by: Alexey Rusakov --- api/client-server/content-repo.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index f76b2fb9..943fee3c 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -100,6 +100,7 @@ paths: name: allow_remote x-example: false required: false + default: true description: | Indicates to the server that it should not attempt to fetch the media if it is deemed remote. This is to prevent routing loops where the server contacts itself. Defaults to @@ -154,6 +155,7 @@ paths: name: allow_remote x-example: false required: false + default: true description: | Indicates to the server that it should not attempt to fetch the media if it is deemed remote. This is to prevent routing loops where the server contacts itself. Defaults to @@ -221,6 +223,7 @@ paths: name: allow_remote x-example: false required: false + default: true description: | Indicates to the server that it should not attempt to fetch the media if it is deemed remote. This is to prevent routing loops where the server contacts itself. Defaults to From ee98082a6051a4871a1936bd2ddf7d3d97729980 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Mon, 4 Jun 2018 12:12:48 +0100 Subject: [PATCH 067/308] update MSC1270 --- scripts/proposals.py | 2 ++ specification/proposals.rst | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) mode change 100644 => 100755 scripts/proposals.py diff --git a/scripts/proposals.py b/scripts/proposals.py old mode 100644 new mode 100755 index 241b0708..aa2252bc --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -1,3 +1,5 @@ +#! /usr/bin/env python + # proposals.py: generate an RST file (proposals.rst) from queries to github.com/matrix.org/matrix-doc/issues. # v0.0.1 diff --git a/specification/proposals.rst b/specification/proposals.rst index 636877ff..9e0becf3 100644 --- a/specification/proposals.rst +++ b/specification/proposals.rst @@ -229,9 +229,9 @@ proposal-ready-for-review - None - * - `MSC1270 `_ - - Synapse /_matrix/media/v1/resolve_url to API download and preview urls in the clients despite on CORS - - 2018-05-31 + - Add /_matrix/media/v1/resolve_url to Client-Server API: download and preview urls in the clients despite CORS - 2018-05-31 + - 2018-06-04 - `1270-1 `_ - `@oivoodoo`_ - None @@ -376,7 +376,7 @@ spec-pr-missing * - `MSC739 `_ - Reporting inappropriate content in Matrix - 2016-11-21 - - 2018-05-15 + - 2018-05-31 - `739-1 `_ - `@ara4n`_ - None From 4ff5887c917b3595344f01d33b134a1658a1e4a1 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 5 Jun 2018 10:06:26 +0100 Subject: [PATCH 068/308] spec the highlights field in /search response --- api/client-server/search.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/client-server/search.yaml b/api/client-server/search.yaml index c4f38fa1..7c5a9a5b 100644 --- a/api/client-server/search.yaml +++ b/api/client-server/search.yaml @@ -181,6 +181,12 @@ paths: count: type: number description: An approximate count of the total number of results found. + highlights: + type: array + title: Highlights + description: List of words which should be highlighted, useful for stemming which may change the query terms. + items: + type: string results: type: array title: Results From 64f97c5b7861b84bd53993214c89db6dff97862b Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 5 Jun 2018 10:10:24 +0100 Subject: [PATCH 069/308] add example of highlights field in /search response --- api/client-server/search.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/client-server/search.yaml b/api/client-server/search.yaml index 7c5a9a5b..65730e38 100644 --- a/api/client-server/search.yaml +++ b/api/client-server/search.yaml @@ -324,6 +324,10 @@ paths: } } }, + "highlights": [ + "martians", + "men" + ], "next_batch": "5FdgFsd234dfgsdfFD", "count": 1224, "results": [ From df5455de1f993104c772f85af6f115ec0751a3d1 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 5 Jun 2018 10:12:34 +0100 Subject: [PATCH 070/308] re-arrange fields in example /search request to make them a valid request. --- api/client-server/search.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/api/client-server/search.yaml b/api/client-server/search.yaml index 65730e38..e5f6a872 100644 --- a/api/client-server/search.yaml +++ b/api/client-server/search.yaml @@ -53,16 +53,16 @@ paths: "keys": [ "content.body" ], - "search_term": "martians and men" - } - }, - "order_by": "recent", - "groupings": { - "group_by": [ - { - "key": "room_id" + "search_term": "martians and men", + "order_by": "recent", + "groupings": { + "group_by": [ + { + "key": "room_id" + } + ] } - ] + } } } properties: From 968ee0f61379b5c021ef26044ba59e566906014e Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 5 Jun 2018 10:13:54 +0100 Subject: [PATCH 071/308] add changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 92d13103..24344116 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -54,6 +54,8 @@ Unreleased changes (`#1264 `_). - Document ``/logout/all`` endpoint (`#1263 `_). + - Document `highlights` field in /search response + (`#1274 `_). r0.3.0 ====== From ffb8037af2d1a1e7e7657f8f7154c5175093034b Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Tue, 5 Jun 2018 16:43:42 +0100 Subject: [PATCH 072/308] update MSC list --- specification/proposals.rst | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/specification/proposals.rst b/specification/proposals.rst index 9e0becf3..6c41a0ac 100644 --- a/specification/proposals.rst +++ b/specification/proposals.rst @@ -49,6 +49,14 @@ proposal-wip - `@leonerd`_ - None - + * - `MSC1148 `_ + - Support for websockets + - 2015-11-16 + - 2018-06-04 + - `1148-1 `_, `1148-2 `_ + - `@richvdh`_, `@krombel`_ + - None + - * - `MSC1238 `_ - Push to Talk - 2016-04-13 @@ -193,6 +201,14 @@ proposal-wip - `@uhoreg`_ - None - + * - `MSC1280 `_ + - Mechanisms for communicating erasure requests to bots and federated homeservers + - 2018-06-05 + - 2018-06-05 + - `1280-1 `_ + - `@richvdh`_ + - None + - @@ -215,7 +231,7 @@ proposal-ready-for-review * - `MSC1232 `_ - Media limits API - 2018-05-04 - - 2018-05-15 + - 2018-06-05 - `1232-1 `_ - `@Half-Shot`_ - None @@ -231,7 +247,7 @@ proposal-ready-for-review * - `MSC1270 `_ - Add /_matrix/media/v1/resolve_url to Client-Server API: download and preview urls in the clients despite CORS - 2018-05-31 - - 2018-06-04 + - 2018-06-05 - `1270-1 `_ - `@oivoodoo`_ - None @@ -636,14 +652,15 @@ obsolete .. _@rxl881: https://github.com/rxl881 -.. _@psaavedra: https://github.com/psaavedra +.. _@turt2live: https://github.com/turt2live .. _@erikjohnston: https://github.com/erikjohnston .. _@t3chguy: https://github.com/t3chguy .. _@Kegsay: https://github.com/Kegsay .. _@KitsuneRal: https://github.com/KitsuneRal .. _@leonerd: https://github.com/leonerd -.. _@turt2live: https://github.com/turt2live +.. _@psaavedra: https://github.com/psaavedra .. _@ara4n: https://github.com/ara4n +.. _@krombel: https://github.com/krombel .. _@maxidor: https://github.com/maxidor .. _@uhoreg: https://github.com/uhoreg .. _@pik: https://github.com/pik From cefd1a832e1849793917caffe89f417c01c2ac99 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 5 Jun 2018 17:55:21 +0100 Subject: [PATCH 073/308] server_server_api: fix typo in auth section fixes "destintation" typo --- specification/server_server_api.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index e9885311..9ad04d10 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -18,7 +18,8 @@ Federation API Matrix homeservers use the Federation APIs (also known as server-server APIs) to communicate with each other. Homeservers use these APIs to push messages to -each other in real-time, to request historic messages from each other, and to +each other in real-time, to +historic messages from each other, and to query profile and presence information about users on each other's servers. The APIs are implemented using HTTPS GETs and PUTs between each of the @@ -1173,7 +1174,7 @@ Step 1 sign JSON: "method": "GET", "uri": "/target", "origin": "origin.hs.example.com", - "destintation": "destination.hs.example.com", + "destination": "destination.hs.example.com", "content": , "signatures": { "origin.hs.example.com": { From a552c1853e10c660f0ded997d43dfe5fdd80f45b Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Thu, 7 Jun 2018 14:07:02 +0100 Subject: [PATCH 074/308] update for MSC 701 --- specification/proposals.rst | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/specification/proposals.rst b/specification/proposals.rst index 6c41a0ac..2b3e3e63 100644 --- a/specification/proposals.rst +++ b/specification/proposals.rst @@ -252,6 +252,14 @@ proposal-ready-for-review - `@oivoodoo`_ - None - + * - `MSC701 `_ + - Auth for content repo (and enforcing GDPR erasure) + - 2018-06-04 + - 2018-06-07 + - `701-1 `_ + - `@ara4n`_ + - None + - @@ -280,9 +288,9 @@ proposal-in-review - `@uhoreg`_ - * - `MSC1194 `_ - - A way for HSes to remove bindings from ISes + - A way for HSes to remove bindings from ISes (aka unbind) - 2018-05-09 - - 2018-05-15 + - 2018-06-05 - `1194-1 `_ - `@dbkr`_ - None From 0c1ba33ebadf45e65234eb8017e542ba020e958a Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Thu, 7 Jun 2018 14:30:50 +0100 Subject: [PATCH 075/308] include proposals.py in matrix-doc CI --- scripts/generate-matrix-org-assets | 3 +++ scripts/proposals.py | 3 +-- scripts/requirements.txt | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/generate-matrix-org-assets b/scripts/generate-matrix-org-assets index cb3cf455..ed08f81d 100755 --- a/scripts/generate-matrix-org-assets +++ b/scripts/generate-matrix-org-assets @@ -8,6 +8,9 @@ cd `dirname $0`/.. mkdir -p assets +# generate specification/proposals.rst +./scripts/proposals.py + # generate the spec docs ./scripts/gendoc.py -d assets/spec diff --git a/scripts/proposals.py b/scripts/proposals.py index aa2252bc..b489f0d2 100755 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -6,7 +6,6 @@ import requests import re from datetime import datetime -from m2r import convert as m2r pagecount = 1 authors = set() @@ -42,7 +41,7 @@ issues = {} for label in labels: issues[label] = getbylabel(label) -text_file = open("../specification/proposals.rst", "w") +text_file = open("specification/proposals.rst", "w") text_file.write("Tables of Tracked Proposals\n---------------------------\n\n") diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 9886fce4..37d99dd3 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -6,3 +6,4 @@ pygments >= 2.2.0 Jinja2 >= 2.9.6 jsonschema >= 2.6.0 PyYAML >= 3.12 +requests >= 2.18.4 From 18ba0d6d81c536db1d47b75301b9076a40683772 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Thu, 7 Jun 2018 15:32:18 +0100 Subject: [PATCH 076/308] remove propsals.rst as it will be regenerated --- specification/proposals.rst | 689 ------------------------------------ 1 file changed, 689 deletions(-) delete mode 100644 specification/proposals.rst diff --git a/specification/proposals.rst b/specification/proposals.rst deleted file mode 100644 index 2b3e3e63..00000000 --- a/specification/proposals.rst +++ /dev/null @@ -1,689 +0,0 @@ -Tables of Tracked Proposals ---------------------------- - -proposal-wip -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC455 `_ - - Do we want to specify a matrix:// URI scheme for rooms? (SPEC-5) - - 2014-09-15 - - 2018-05-15 - - `455-1 `_ - - `@KitsuneRal`_ - - None - - - * - `MSC586 `_ - - Federation API for canonicalising MXIDs - - 2014-10-27 - - 2018-05-15 - - `586-1 `_ - - `@ara4n`_ - - None - - - * - `MSC489 `_ - - Extensible Profiles. (SPEC-93) - - 2015-01-19 - - 2018-05-15 - - `489-1 `_ - - `@erikjohnston`_ - - None - - - * - `MSC1196 `_ - - Matrix Escape Hatch (Versioned Rooms) - - 2015-10-22 - - 2018-05-15 - - `1196-1 `_ - - `@leonerd`_ - - None - - - * - `MSC1148 `_ - - Support for websockets - - 2015-11-16 - - 2018-06-04 - - `1148-1 `_, `1148-2 `_ - - `@richvdh`_, `@krombel`_ - - None - - - * - `MSC1238 `_ - - Push to Talk - - 2016-04-13 - - 2018-05-15 - - `1238-1 `_ - - `@aviraldg`_ - - None - - `PR#310`_ - * - `MSC1198 `_ - - Threading API - - 2016-05-23 - - 2018-05-15 - - `1198-1 `_ - - `@Half-Shot`_ - - None - - - * - `MSC1207 `_ - - Publishing Room Lists for 3rd party networks - - 2016-10-21 - - 2018-05-31 - - `1207-1 `_ - - `@erikjohnston`_ - - None - - - * - `MSC441 `_ - - Support for Reactions / Aggregations - - 2016-12-25 - - 2018-05-15 - - `441-1 `_ - - `@pik`_ - - `@ara4n`_ - - - * - `MSC1237 `_ - - Improving m.location with GeoJSON and accuracy - - 2017-05-19 - - 2018-05-15 - - `1237-1 `_ - - `@Half-Shot`_ - - None - - `PR#919`_ - * - `MSC971 `_ - - Add groups stuff to spec - - 2017-08-10 - - 2018-05-20 - - `971-1 `_, `971-2 `_, `971-3 `_ - - `@erikjohnston`_ - - None - - - * - `MSC1215 `_ - - Groups as Rooms - - 2017-10-17 - - 2018-05-15 - - `1215-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1217 `_ - - Visibility of groups to group members and their non-members - - 2017-10-30 - - 2018-05-15 - - `1217-1 `_ - - `@lampholder`_ - - None - - - * - `MSC1218 `_ - - Bridging group membership with 3rd party group sources - - 2017-11-15 - - 2018-05-15 - - `1218-1 `_ - - `@lukebarnard1`_ - - None - - - * - `MSC1219 `_ - - Proposal for storing megolm keys serverside - - 2017-11-23 - - 2018-05-15 - - `1219-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1221 `_ - - Improving Presence - - 2017-12-26 - - 2018-05-24 - - `1221-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1222 `_ - - Pushing updates about Groups (Communities) to clients - - 2018-01-02 - - 2018-05-24 - - `1222-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1225 `_ - - Extensible event types & fallback in Matrix - - 2018-02-18 - - 2018-05-15 - - `1225-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1227 `_ - - Proposal for lazy-loading room members to improve initial sync speed and client RAM usage - - 2018-03-05 - - 2018-05-28 - - `1227-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1206 `_ - - Proposal for improved bot support - - 2018-03-14 - - 2018-05-15 - - `1206-1 `_ - - `@turt2live`_ - - None - - - * - `MSC1228 `_ - - Removing MXIDs from events - - 2018-04-19 - - 2018-05-15 - - `1228-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1231 `_ - - Handling Consent for Privacy Policy - - 2018-05-02 - - 2018-05-15 - - `1231-1 `_ - - `@neilisfragile`_ - - None - - - * - `MSC1267 `_ - - Interactive Key Verification - - 2018-05-28 - - 2018-05-28 - - `1267-1 `_ - - `@uhoreg`_ - - None - - - * - `MSC1280 `_ - - Mechanisms for communicating erasure requests to bots and federated homeservers - - 2018-06-05 - - 2018-06-05 - - `1280-1 `_ - - `@richvdh`_ - - None - - - - - -proposal-ready-for-review -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC1232 `_ - - Media limits API - - 2018-05-04 - - 2018-06-05 - - `1232-1 `_ - - `@Half-Shot`_ - - None - - `PR#1189`_ - * - `MSC1256 `_ - - Custom emoji and sticker packs in matrix - - 2018-05-23 - - 2018-05-24 - - `1256-1 `_ - - `@turt2live`_ - - None - - - * - `MSC1270 `_ - - Add /_matrix/media/v1/resolve_url to Client-Server API: download and preview urls in the clients despite CORS - - 2018-05-31 - - 2018-06-05 - - `1270-1 `_ - - `@oivoodoo`_ - - None - - - * - `MSC701 `_ - - Auth for content repo (and enforcing GDPR erasure) - - 2018-06-04 - - 2018-06-07 - - `701-1 `_ - - `@ara4n`_ - - None - - - - - -proposal-in-review -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC433 `_ - - Support for discovering API endpoints via .well-known URIs (SPEC-121) - - 2015-03-08 - - 2018-05-15 - - `433-1 `_, `433-2 `_ - - `@maxidor`_, `others`_ - - `@uhoreg`_ - - - * - `MSC1194 `_ - - A way for HSes to remove bindings from ISes (aka unbind) - - 2018-05-09 - - 2018-06-05 - - `1194-1 `_ - - `@dbkr`_ - - None - - - - - -proposal-passed-review -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC1226 `_ - - State Reset mitigation proposal - - 2018-02-20 - - 2018-05-15 - - `1226-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1229 `_ - - Mitigating abuse of the event depth parameter over federation - - 2018-04-30 - - 2018-05-15 - - `1229-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1236 `_ - - Matrix Widget API v2 - - 2018-05-13 - - 2018-05-15 - - `1236-1 `_ - - `@rxl881`_ - - None - - - - - -spec-pr-missing -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC1200 `_ - - Configuration of E2E encryption in a room - - 2016-06-16 - - 2018-05-31 - - `1200-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1201 `_ - - Device Management API - - 2016-07-14 - - 2018-05-15 - - `1201-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1203 `_ - - 3rd Party Entity lookup API - - 2016-07-21 - - 2018-05-15 - - `1203-1 `_ - - `@leonerd`_ - - None - - - * - `MSC1205 `_ - - Proposal for multi-device deletion API - - 2016-10-12 - - 2018-05-15 - - `1205-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1208 `_ - - Encrypted attachment format - - 2016-10-26 - - 2018-05-15 - - `1208-1 `_ - - `@NegativeMjark`_ - - None - - - * - `MSC739 `_ - - Reporting inappropriate content in Matrix - - 2016-11-21 - - 2018-05-31 - - `739-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1211 `_ - - Megolm session export format - - 2017-01-03 - - 2018-05-15 - - `1211-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1212 `_ - - Device List Update Stream - - 2017-01-18 - - 2018-05-15 - - `1212-1 `_ - - `@richvdh`_ - - None - - - * - `MSC829 `_ - - Need to spec msisdn login API - - 2017-03-08 - - 2018-05-15 - - `829-1 `_ - - `@dbkr`_ - - None - - - * - `MSC855 `_ - - spec m.login.msisdn UI auth type - - 2017-03-24 - - 2018-05-15 - - `855-1 `_ - - `@dbkr`_ - - None - - - * - `MSC910 `_ - - Add new Read Marker API to docs - - 2017-05-08 - - 2018-05-15 - - - - `@lukebarnard1`_ - - None - - - * - `MSC1067 `_ - - Spec @mentions - - 2017-07-14 - - 2018-05-15 - - `1067-1 `_ - - `@lukebarnard1`_ - - None - - - * - `MSC1214 `_ - - Related Groups (i.e. flair) - - 2017-10-03 - - 2018-05-15 - - `1214-1 `_ - - `@lukebarnard1`_ - - None - - - * - `MSC1033 `_ - - Doc @room notifications - - 2017-10-23 - - 2018-05-31 - - - - `@dbkr`_ - - None - - - * - `MSC1183 `_ - - Document key-share requests - - 2018-04-30 - - 2018-05-31 - - `1183-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1230 `_ - - Temporary mitigation for depth parameter abuse - - 2018-05-01 - - 2018-05-15 - - `1230-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1234 `_ - - Rich Replies format - - 2018-05-12 - - 2018-05-18 - - `1234-1 `_ - - `@t3chguy`_ - - None - - - - - -merged -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC1197 `_ - - Ignoring Users - - 2016-05-03 - - 2018-05-18 - - `1197-1 `_ - - `@erikjohnston`_ - - None - - `PR#1142`_ - * - `MSC1199 `_ - - Notifications API - - 2016-05-23 - - 2018-05-15 - - `1199-1 `_ - - `@dbkr`_ - - None - - - * - `MSC1204 `_ - - Access Token Semantics (refresh and macaroons) - aka Auth Sept 2016 Edition - - 2016-09-29 - - 2018-05-15 - - `1204-1 `_ - - `@richvdh`_ - - None - - - * - `MSC953 `_ - - Add /user_directory/search API - - 2017-05-31 - - 2018-05-10 - - `953-1 `_ - - `@erikjohnston`_ - - None - - - * - `MSC1233 `_ - - A proposal for organising spec proposals - - 2018-05-10 - - 2018-05-18 - - `1233-1 `_ - - `@ara4n`_ - - None - - `PR#1240`_ - - - -abandoned -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC531 `_ - - Homeservers as OAuth authorization endpoints (resource owners) (SPEC-206) - - 2015-07-25 - - 2018-05-15 - - `531-1 `_ - - `@Kegsay`_ - - None - - - * - `MSC1202 `_ - - Profile Personae - - 2016-07-15 - - 2018-05-15 - - `1202-1 `_ - - `@erikjohnston`_ - - None - - - * - `MSC1209 `_ - - Server Side Profile API - - 2016-11-01 - - 2018-05-15 - - `1209-1 `_ - - `@erikjohnston`_ - - None - - - * - `MSC1213 `_ - - Set defaults for m.federate - - 2017-04-10 - - 2018-05-18 - - `1213-1 `_ - - `@psaavedra`_ - - None - - - - - -obsolete -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC1223 `_ - - Replies event format - - 2018-02-01 - - 2018-05-15 - - `1223-1 `_ - - `@t3chguy`_ - - None - - - * - `MSC1224 `_ - - Replies - next steps - - 2018-02-03 - - 2018-05-15 - - `1224-1 `_ - - `@t3chguy`_ - - None - - - * - `MSC1235 `_ - - Proposal for Calendar Events - - 2018-02-06 - - 2018-05-15 - - `1235-1 `_ - - `@Half-Shot`_ - - None - - - * - `MSC1220 `_ - - Rich quoting proposal - - 2018-05-10 - - 2018-05-15 - - `1220-1 `_ - - `@t3chguy`_ - - None - - - - - - - -.. _@rxl881: https://github.com/rxl881 -.. _@turt2live: https://github.com/turt2live -.. _@erikjohnston: https://github.com/erikjohnston -.. _@t3chguy: https://github.com/t3chguy -.. _@Kegsay: https://github.com/Kegsay -.. _@KitsuneRal: https://github.com/KitsuneRal -.. _@leonerd: https://github.com/leonerd -.. _@psaavedra: https://github.com/psaavedra -.. _@ara4n: https://github.com/ara4n -.. _@krombel: https://github.com/krombel -.. _@maxidor: https://github.com/maxidor -.. _@uhoreg: https://github.com/uhoreg -.. _@pik: https://github.com/pik -.. _@neilisfragile: https://github.com/neilisfragile -.. _@lukebarnard1: https://github.com/lukebarnard1 -.. _others: https://github.com/thers -.. _@Half-Shot: https://github.com/Half-Shot -.. _@aviraldg: https://github.com/aviraldg -.. _@oivoodoo: https://github.com/oivoodoo -.. _@richvdh: https://github.com/richvdh -.. _@NegativeMjark: https://github.com/NegativeMjark -.. _@lampholder: https://github.com/lampholder -.. _@dbkr: https://github.com/dbkr -.. _PR#310: https://github.com/matrix-org/matrix-doc/pull/310 -.. _PR#1142: https://github.com/matrix-org/matrix-doc/pull/1142 -.. _PR#1240: https://github.com/matrix-org/matrix-doc/pull/1240 -.. _PR#1189: https://github.com/matrix-org/matrix-doc/pull/1189 -.. _PR#919: https://github.com/matrix-org/matrix-doc/pull/919 \ No newline at end of file From 72282577bf5f827fc052a623c6109ee79eda8438 Mon Sep 17 00:00:00 2001 From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com> Date: Thu, 7 Jun 2018 16:32:03 +0100 Subject: [PATCH 077/308] Add back proposals.rst with a link to the actual location --- specification/proposals.rst | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 specification/proposals.rst diff --git a/specification/proposals.rst b/specification/proposals.rst new file mode 100644 index 00000000..371850ab --- /dev/null +++ b/specification/proposals.rst @@ -0,0 +1,6 @@ +Tables of Tracked Proposals +--------------------------- + +This file is autogenerated by a jenkins build process + +View the current live version `at https://matrix.org/docs/spec/proposals `_ From c3f66e18147d4139ebc02ffae64cadc5217211a6 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Mon, 11 Jun 2018 12:40:04 +0100 Subject: [PATCH 078/308] basic spec for group IDs, mostly copied from User IDs as they share grammar --- .../appendices/identifier_grammar.rst | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/specification/appendices/identifier_grammar.rst b/specification/appendices/identifier_grammar.rst index e85bf410..9487374b 100644 --- a/specification/appendices/identifier_grammar.rst +++ b/specification/appendices/identifier_grammar.rst @@ -60,6 +60,7 @@ The sigil characters are as follows: * ``@``: User ID * ``!``: Room ID * ``$``: Event ID +* ``+``: Group ID * ``#``: Room alias The precise grammar defining the allowable format of an identifier depends on @@ -207,6 +208,35 @@ readable. .. TODO-spec What is the grammar for the opaque part? https://matrix.org/jira/browse/SPEC-389 + +Group Identifiers ++++++++++++++++++ + +Groups within Matrix are uniquely identified by their group ID. The group +ID is namespaced to the group server which hosts this group and has the +form:: + + +localpart:domain + +The ``localpart`` of a group ID is an opaque identifier for that group. It MUST +NOT be empty, and MUST contain only the characters ``a-z``, ``0-9``, ``.``, +``_``, ``=``, ``-``, and ``/``. + +The ``domain`` of a group ID is the `server name`_ of the group server which +hosts this group. + +The length of a group ID, including the ``+`` sigil and the domain, MUST NOT +exceed 255 characters. + +The complete grammar for a legal group ID is:: + + group_id = "+" group_id_localpart ":" server_name + group_id_localpart = 1*group_id_char + group_id_char = DIGIT + / %x61-7A ; a-z + / "-" / "." / "=" / "_" / "/" + + Room Aliases ++++++++++++ From 2e5cdaf51c28103f6aa30866beb5855b28499559 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 11 Jun 2018 14:25:05 +0100 Subject: [PATCH 079/308] Fix typo "To stream events all the events" -> "To stream all the events" --- specification/server_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 9ad04d10..8d6c8eba 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -781,7 +781,7 @@ room. Starting from the PDU ID(s) given in the "v" argument, the PDUs that preceded it are retrieved, up to a total number given by the "limit" argument. -To stream events all the events:: +To stream all the events:: GET .../pull/ Query args: origin, v From 5ab5fe08d1b56c58e073e587b42928923ac41f98 Mon Sep 17 00:00:00 2001 From: user Date: Sun, 17 Jun 2018 15:38:35 +0100 Subject: [PATCH 080/308] Small grammatical and stylistic fixes --- api/server-server/directory.yaml | 6 +- scripts/css/blockquote.css | 1 - specification/server_server_api.rst | 420 +++++++++++++++------------- 3 files changed, 221 insertions(+), 206 deletions(-) diff --git a/api/server-server/directory.yaml b/api/server-server/directory.yaml index c4a06231..516b1bf8 100644 --- a/api/server-server/directory.yaml +++ b/api/server-server/directory.yaml @@ -27,13 +27,13 @@ paths: get: summary: Retrieve the room ID and list of resident homeservers for a room alias. - description: Retrieve the room ID and list of resident homeservers for a Room + description: Retrieve the room ID and list of resident homeservers for a room alias. parameters: - in: query name: room_alias type: string - description: Room alias + description: Room alias. required: true x-example: "#room_alias:example.org" responses: @@ -65,4 +65,4 @@ paths: "example.com", "another.example.com:8449", ] - } \ No newline at end of file + } diff --git a/scripts/css/blockquote.css b/scripts/css/blockquote.css index 151d3bce..05fa73bc 100644 --- a/scripts/css/blockquote.css +++ b/scripts/css/blockquote.css @@ -1,5 +1,4 @@ blockquote { margin: 20px 0 30px; - border-left: 5px solid; padding-left: 20px; } diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 8d6c8eba..e6484853 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -126,20 +126,20 @@ Publishing Keys Homeservers publish the allowed TLS fingerprints and signing keys in a JSON object at ``/_matrix/key/v2/server/{key_id}``. The response contains a list of ``verify_keys`` that are valid for signing federation requests made by the -server and for signing events. It contains a list of ``old_verify_keys`` -which are only valid for signing events. Finally the response contains a list -of TLS certificate fingerprints to validate any connection made to the server. +server and for signing events. It contains a list of ``old_verify_keys`` which +are only valid for signing events. Finally the response contains a list of TLS +certificate fingerprints to validate any connection made to the server. A server may have multiple keys active at a given time. A server may have any number of old keys. It is recommended that servers return a single JSON response listing all of its keys whenever any ``key_id`` is requested to reduce the number of round trips needed to discover the relevant keys for a server. -However a server may return a different responses for a different ``key_id``. +However a server may return different responses for a different ``key_id``. -The ``tls_certificates`` contain a list of hashes of the X.509 TLS certificates -currently used by the server. The list must include SHA-256 hashes for every -certificate currently in use by the server. These fingerprints are valid until -the millisecond POSIX timestamp in ``valid_until_ts``. +The ``tls_certificates`` field contains a list of hashes of the X.509 TLS +certificates currently used by the server. The list must include SHA-256 hashes +for every certificate currently in use by the server. These fingerprints are +valid until the millisecond POSIX timestamp in ``valid_until_ts``. The ``verify_keys`` can be used to sign requests and events made by the server until the millisecond POSIX timestamp in ``valid_until_ts``. If a homeserver @@ -152,17 +152,18 @@ before the ``expired_ts``. The ``expired_ts`` is a millisecond POSIX timestamp of when the originating server stopped using that key. Intermediate notary servers should cache a response for half of its remaining -life time to avoid serving a stale response. Originating servers should avoid +lifetime to avoid serving a stale response. Originating servers should avoid returning responses that expire in less than an hour to avoid repeated requests -for an about to expire certificate. Requesting servers should limit how -frequently they query for certificates to avoid flooding a server with requests. +for a certificate that is about to expire. Requesting servers should limit how +frequently they query for certificates to avoid flooding a server with +requests. If a server goes offline intermediate notary servers should continue to return the last response they received from that server so that the signatures of old events sent by that server can still be checked. ==================== =================== ====================================== - Key Type Description + Key Type Description ==================== =================== ====================================== ``server_name`` String DNS name of the homeserver. ``verify_keys`` Object Public keys of the homeserver for @@ -172,7 +173,7 @@ events sent by that server can still be checked. ``signatures`` Object Digital signatures for this object signed using the ``verify_keys``. ``tls_fingerprints`` Array of Objects Hashes of X.509 TLS certificates used - by this this server encoded as `Unpadded Base64`_. + by this server encoded as `Unpadded Base64`_. ``valid_until_ts`` Integer POSIX timestamp when the list of valid keys should be refreshed. ==================== =================== ====================================== @@ -209,38 +210,38 @@ events sent by that server can still be checked. Querying Keys Through Another Server ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Servers may offer a query API ``_matrix/key/v2/query/`` for getting the keys -for another server. This API can be used to GET at list of JSON objects for a +Servers may offer a query API ``/_matrix/key/v2/query/`` for getting the keys +for another server. This API can be used to GET a list of JSON objects for a given server or to POST a bulk query for a number of keys from a number of servers. Either way the response is a list of JSON objects containing the -JSON published by the server under ``_matrix/key/v2/server/`` signed by +JSON published by the server under ``/_matrix/key/v2/server/`` signed by both the originating server and by this server. The ``minimum_valid_until_ts`` is a millisecond POSIX timestamp indicating when the returned certificate will need to be valid until to be useful to the requesting server. This can be set using the maximum ``origin_server_ts`` of -an batch of events that a requesting server is trying to validate. This allows +a batch of events that a requesting server is trying to validate. This allows an intermediate notary server to give a prompt cached response even if the originating server is offline. -This API can return keys for servers that are offline be using cached responses +This API can return keys for servers that are offline by using cached responses taken from when the server was online. Keys can be queried from multiple servers to mitigate against DNS spoofing. -Requests: +Example request: .. code:: - GET /_matrix/key/v2/query/${server_name}/${key_id}/?minimum_valid_until_ts=${minimum_valid_until_ts} HTTP/1.1 + GET /_matrix/key/v2/query/{server_name}/{key_id}/?minimum_valid_until_ts={minimum_valid_until_ts} HTTP/1.1 POST /_matrix/key/v2/query HTTP/1.1 Content-Type: application/json { "server_keys": { - "$server_name": { - "$key_id": { - "minimum_valid_until_ts": $posix_timestamp + "{server_name}": { + "{key_id}": { + "minimum_valid_until_ts": {posix_timestamp} } } } @@ -263,14 +264,14 @@ Response: Version 1 +++++++++ .. WARNING:: - Version 1 of key distribution is obsolete + Version 1 of key distribution is obsolete. Homeservers publish their TLS certificates and signing keys in a JSON object at ``/_matrix/key/v1``. ==================== =================== ====================================== - Key Type Description + Key Type Description ==================== =================== ====================================== ``server_name`` String DNS name of the homeserver. ``verify_keys`` Object Public keys of the homeserver for @@ -299,7 +300,7 @@ at ``/_matrix/key/v1``. When fetching the keys for a server the client should check that the TLS certificate in the JSON matches the TLS server certificate for the connection and should check that the JSON signatures are correct for the supplied -``verify_keys`` +``verify_keys``. Transactions ------------ @@ -321,7 +322,7 @@ Transaction Fields ~~~~~~~~~~~~~~~~~~ ==================== =================== ====================================== - Key Type Description + Key Type Description ==================== =================== ====================================== ``origin`` String **Required**. ``server_name`` of homeserver sending this transaction. @@ -339,10 +340,10 @@ Example: .. code:: json { - "origin_server_ts":1404835423000, - "origin":"matrix.org", - "pdus":[...], - "edus":[...] + "origin_server_ts": 1404835423000, + "origin": "matrix.org", + "pdus": [...], + "edus": [...] } PDUs @@ -368,19 +369,18 @@ PDU Fields ``origin_server_ts`` Integer **Required**. Timestamp in milliseconds on origin homeserver when this event was created. -``type`` String **Required**. Event type -``state_key`` String Optional. If this key is present, the - event is a state event, and it will - replace previous events with the same - ``type`` and ``state_key`` in the room - state. +``type`` String **Required**. Event type. +``state_key`` String If this key is present, the event is a + state event, and it will replace + previous events with the same ``type`` + and ``state_key`` in the room state. ``content`` Object **Required**. The content of the event. ``prev_events`` List of (String, **Required**. Event IDs and hashes of {String: String}) the most recent events in the room that pairs the homeserver was aware of when it - made this event + made this event. ``depth`` Integer **Required**. The maximum depth of the - ``prev_events``, plus one + ``prev_events``, plus one. ``auth_events`` List of (String, **Required**. Event IDs and hashes for {String: String}) the "auth events" of this event. pairs @@ -390,10 +390,10 @@ PDU Fields ``signatures`` {String: **Required**. Signatures of the redacted {String: String}} PDU, following the algorithm specified in `Signing Events`_. -``redacts`` String Optional. For redaction events, the ID - of the event being redacted -``unsigned`` Object Optional. Additional data added by the - origin server but not covered by the +``redacts`` String For redaction events, the ID of the + event being redacted. +``unsigned`` Object Additional data added by the origin + server but not covered by the ``signatures``. ==================== ================== ======================================= @@ -500,119 +500,121 @@ the state of the room. state of the room. For example, a redacted ``join`` event will still result in the user being considered joined. +The rules are as follows: + 1. If type is ``m.room.create``, allow if and only if it has no previous events - *i.e.* it is the first event in the room. -#. If type is ``m.room.member``: +2. If type is ``m.room.member``: - a. If ``membership`` is ``join``: + a. If ``membership`` is ``join``: - i. If the only previous event is an ``m.room.create`` - and the ``state_key`` is the creator, allow. + i. If the only previous event is an ``m.room.create`` + and the ``state_key`` is the creator, allow. - #. If the ``sender`` does not match ``state_key``, reject. + #. If the ``sender`` does not match ``state_key``, reject. - #. If the user's current membership state is ``invite`` or ``join``, - allow. + #. If the user's current membership state is ``invite`` or ``join``, + allow. - #. If the ``join_rule`` is ``public``, allow. + #. If the ``join_rule`` is ``public``, allow. - #. Otherwise, reject. + #. Otherwise, reject. - #. If ``membership`` is ``invite``: + b. If ``membership`` is ``invite``: - i. If the ``sender``'s current membership state is not ``join``, reject. + i. If the ``sender``'s current membership state is not ``join``, reject. - #. If *target user*'s current membership state is ``join`` or ``ban``, - reject. + #. If *target user*'s current membership state is ``join`` or ``ban``, + reject. - #. If the ``sender``'s power level is greater than or equal to the *invite - level*, allow. + #. If the ``sender``'s power level is greater than or equal to the *invite + level*, allow. - #. Otherwise, reject. + #. Otherwise, reject. - #. If ``membership`` is ``leave``: + c. If ``membership`` is ``leave``: - i. If the ``sender`` matches ``state_key``, allow if and only if that user's - current membership state is ``invite`` or ``join``. + i. If the ``sender`` matches ``state_key``, allow if and only if that user's + current membership state is ``invite`` or ``join``. - #. If the ``sender``'s current membership state is not ``join``, reject. + #. If the ``sender``'s current membership state is not ``join``, reject. - #. If the *target user*'s current membership state is ``ban``, and the - ``sender``'s power level is less than the *ban level*, reject. + #. If the *target user*'s current membership state is ``ban``, and the + ``sender``'s power level is less than the *ban level*, reject. - #. If the ``sender``'s power level is greater than or equal to the *kick - level*, and the *target user*'s power level is less than the - ``sender``'s power level, allow. + #. If the ``sender``'s power level is greater than or equal to the *kick + level*, and the *target user*'s power level is less than the + ``sender``'s power level, allow. - #. Otherwise, reject. + #. Otherwise, reject. - #. If ``membership`` is ``ban``: + d. If ``membership`` is ``ban``: - i. If the ``sender``'s current membership state is not ``join``, reject. + i. If the ``sender``'s current membership state is not ``join``, reject. - #. If the ``sender``'s power level is greater than or equal to the *ban - level*, and the *target user*'s power level is less than the - ``sender``'s power level, allow. + #. If the ``sender``'s power level is greater than or equal to the *ban + level*, and the *target user*'s power level is less than the + ``sender``'s power level, allow. - #. Otherwise, reject. + #. Otherwise, reject. - #. Otherwise, the membership is unknown. Reject. + e. Otherwise, the membership is unknown. Reject. -#. If the ``sender``'s current membership state is not ``join``, reject. +3. If the ``sender``'s current membership state is not ``join``, reject. -#. If the event type's *required power level* is greater than the ``sender``'s power +4. If the event type's *required power level* is greater than the ``sender``'s power level, reject. -#. If type is ``m.room.power_levels``: +5. If type is ``m.room.power_levels``: - a. If there is no previous ``m.room.power_levels`` event in the room, allow. + a. If there is no previous ``m.room.power_levels`` event in the room, allow. - #. For each of the keys ``users_default``, ``events_default``, - ``state_default``, ``ban``, ``redact``, ``kick``, ``invite``, as well as - each entry being changed under the ``events`` or ``users`` keys: + b. For each of the keys ``users_default``, ``events_default``, + ``state_default``, ``ban``, ``redact``, ``kick``, ``invite``, as well as + each entry being changed under the ``events`` or ``users`` keys: - i. If the current value is higher than the ``sender``'s current power level, - reject. + i. If the current value is higher than the ``sender``'s current power level, + reject. - #. If the new value is higher than the ``sender``'s current power level, - reject. + #. If the new value is higher than the ``sender``'s current power level, + reject. - #. For each entry being changed under the ``users`` key, other than the - ``sender``'s own entry: + c. For each entry being changed under the ``users`` key, other than the + ``sender``'s own entry: - i. If the current value is equal to the ``sender``'s current power level, - reject. + i. If the current value is equal to the ``sender``'s current power level, + reject. - #. Otherwise, allow. + d. Otherwise, allow. -#. If type is ``m.room.redaction``: +6. If type is ``m.room.redaction``: - #. If the ``sender``'s power level is greater than or equal to the *redact - level*, allow. + a. If the ``sender``'s power level is greater than or equal to the *redact + level*, allow. - #. If the ``sender`` of the event being redacted is the same as the - ``sender`` of the ``m.room.redaction``, allow. + #. If the ``sender`` of the event being redacted is the same as the + ``sender`` of the ``m.room.redaction``, allow. - #. Otherwise, reject. + #. Otherwise, reject. -#. Otherwise, allow. +7. Otherwise, allow. .. NOTE:: - Some consequences of these rules: + Some consequences of these rules: - * Unless you are a member of the room, the only permitted operations (apart - from the intial create/join) are: joining a public room; accepting or - rejecting an invitation to a room. + * Unless you are a member of the room, the only permitted operations (apart + from the intial create/join) are: joining a public room; accepting or + rejecting an invitation to a room. - * To unban somebody, you must have power level greater than or equal to both - the kick *and* ban levels, *and* greater than the target user's power - level. + * To unban somebody, you must have power level greater than or equal to both + the kick *and* ban levels, *and* greater than the target user's power + level. .. TODO-spec - I think there is some magic about 3pid invites too. + I think there is some magic about 3pid invites too. EDUs ---- @@ -628,16 +630,20 @@ destination homeserver names, and the actual nested content. Key Type Description ======================== ============ ========================================= ``edu_type`` String The type of the ephemeral message. +``origin`` String The server name sending the ephemeral + message. +``destination`` String The server name receiving the ephemeral + message. ``content`` Object Content of the ephemeral message. ======================== ============ ========================================= .. code:: json { - "edu_type":"m.presence", - "origin":"blue", - "destination":"orange", - "content":{...} + "edu_type": "m.presence", + "origin": "blue", + "destination": "orange", + "content": {...} } Room State Resolution @@ -817,14 +823,14 @@ below. Joining Rooms ------------- -When a new user wishes to join room that the user's homeserver already knows +When a new user wishes to join a room that the user's homeserver already knows about, the homeserver can immediately determine if this is allowable by -inspecting the state of the room, and if it is acceptable, it can generate, -sign, and emit a new ``m.room.member`` state event adding the user into that -room. When the homeserver does not yet know about the room it cannot do this +inspecting the state of the room. If it is acceptable, it can generate, sign, +and emit a new ``m.room.member`` state event adding the user into that room. +When the homeserver does not yet know about the room it cannot do this directly. Instead, it must take a longer multi-stage handshaking process by which it first selects a remote homeserver which is already participating in -that room, and uses it to assist in the joining process. This is the remote +that room, and use it to assist in the joining process. This is the remote join handshake. This handshake involves the homeserver of the new member wishing to join @@ -886,44 +892,47 @@ of the information that the joining server will need. Despite its name, this object is not a full event; notably it does not need to be hashed or signed by the resident homeserver. The required fields are: -==================== ======== ============ - Key Type Description -==================== ======== ============ -``type`` String The value ``m.room.member`` -``auth_events`` List An event-reference list containing the - authorization events that would allow this member - to join -``content`` Object The event content -``depth`` Integer (this field must be present but is ignored; it - may be 0) -``origin`` String The name of the resident homeserver -``origin_server_ts`` Integer A timestamp added by the resident homeserver -``prev_events`` List An event-reference list containing the immediate - predecessor events -``room_id`` String The room ID of the room -``sender`` String The user ID of the joining member -``state_key`` String The user ID of the joining member -==================== ======== ============ +======================== ============ ========================================= + Key Type Description +======================== ============ ========================================= +``type`` String The value ``m.room.member``. +``auth_events`` List An event-reference list containing the + authorization events that would allow + this member to join. +``content`` Object The event content. +``depth`` Integer (this field must be present but is + ignored; it may be 0) +``origin`` String The name of the resident homeserver. +``origin_server_ts`` Integer A timestamp added by the resident + homeserver. +``prev_events`` List An event-reference list containing the + immediate predecessor events. +``room_id`` String The room ID of the room. +``sender`` String The user ID of the joining member. +``state_key`` String The user ID of the joining member. +======================== ============ ========================================= The ``content`` field itself must be an object, containing: -============== ====== ============ - Key Type Description -============== ====== ============ -``membership`` String The value ``join`` -============== ====== ============ +======================== ============ ========================================= + Key Type Description +======================== ============ ========================================= +``membership`` String The value ``join``. +======================== ============ ========================================= The joining server now has sufficient information to construct the real join event from these protoevent fields. It copies the values of most of them, adding (or replacing) the following fields: -==================== ======= ============ - Key Type Description -==================== ======= ============ -``event_id`` String A new event ID specified by the joining homeserver -``origin`` String The name of the joining homeserver -``origin_server_ts`` Integer A timestamp added by the joining homeserver -==================== ======= ============ +======================== ============ ========================================= + Key Type Description +======================== ============ ========================================= +``event_id`` String A new event ID specified by the joining + homeserver. +``origin`` String The name of the joining homeserver. +``origin_server_ts`` Integer A timestamp added by the joining + homeserver. +======================== ============ ========================================= This will be a true event, so the joining server should apply the event-signing algorithm to it, resulting in the addition of the ``hashes`` and ``signatures`` @@ -934,20 +943,22 @@ event to an resident homeserver, by using the ``send_join`` endpoint. This is invoked using the room ID and the event ID of the new member event. The resident homeserver then accepts this event into the room's event graph, -and responds to the joining server with the full set of state for the newly- +and responds to the joining server with the full set of state for the newly joined room. This is returned as a two-element list, whose first element is the integer 200, and whose second element is an object which contains the following keys: -============== ===== ============ - Key Type Description -============== ===== ============ -``auth_chain`` List A list of events giving all of the events in the auth - chains for the join event and the events in ``state``. -``state`` List A complete list of the prevailing state events at the - instant just before accepting the new ``m.room.member`` - event. -============== ===== ============ +======================== ============ ========================================= + Key Type Description +======================== ============ ========================================= +``auth_chain`` List A list of events giving all of the events + in the auth chains for the join event and + the events in ``state``. +``state`` List A complete list of the prevailing state + events at the instant just before + accepting the new ``m.room.member`` + event. +======================== ============ ========================================= .. TODO-spec - (paul) I don't really understand why the full auth_chain events are given @@ -977,17 +988,18 @@ specifying how many more events of history before that one to return at most. The response to this request is an object with the following keys: -==================== ======== ============ - Key Type Description -==================== ======== ============ -``pdus`` List A list of events -``origin`` String The name of the resident homeserver -``origin_server_ts`` Integer A timestamp added by the resident homeserver -==================== ======== ============ +======================== ============ ========================================= + Key Type Description +======================== ============ ========================================= +``pdus`` List A list of events. +``origin`` String The name of the resident homeserver. +``origin_server_ts`` Integer A timestamp added by the resident + homeserver. +======================== ============ ========================================= The list of events given in ``pdus`` is returned in reverse chronological order; having the most recent event first (i.e. the event whose event ID is -that requested by the requestor in the ``v`` parameter). +that requested by the requester in the ``v`` parameter). .. TODO-spec Specify (or remark that it is unspecified) how the server handles divergent @@ -996,38 +1008,42 @@ that requested by the requestor in the ``v`` parameter). Inviting to a room ------------------ -When a user wishes to invite an other user to a local room and the other user -is on a different server, the inviting server will send a request to the invited +When a user wishes to invite another user to a local room and the other user is +on a different server, the inviting server will send a request to the invited server:: PUT .../invite/{roomId}/{eventId} The required fields in the JSON body are: -==================== ======== ============ - Key Type Description -==================== ======== ============ -``room_id`` String The room ID of the room. Must be the same as the - room ID specified in the path. -``event_id`` String The ID of the event. Must be the same as the event - ID specified in the path. -``type`` String The value ``m.room.member``. -``auth_events`` List An event-reference list containing the IDs of the - authorization events that would allow this member - to be invited in the room. -``content`` Object The content of the event. -``depth`` Integer The depth of the event. -``origin`` String The name of the inviting homeserver. -``origin_server_ts`` Integer A timestamp added by the inviting homeserver. -``prev_events`` List An event-reference list containing the IDs of the - immediate predecessor events. -``sender`` String The Matrix ID of the user who sent the original - `m.room.third_party_invite`. -``state_key`` String The Matrix ID of the invited user. -``signatures`` Object The signature of the event from the origin server. -``unsigned`` Object An object containing the properties that aren't - part of the signature's computation. -==================== ======== ============ +======================== ============ ========================================= + Key Type Description +======================== ============ ========================================= +``room_id`` String The room ID of the room. Must be the same + as the room ID specified in the path. +``event_id`` String The ID of the event. Must be the same as + the event ID specified in the path. +``type`` String The value ``m.room.member``. +``auth_events`` List An event-reference list containing the + IDs of the authorization events that + would allow this member to be invited in + the room. +``content`` Object The content of the event. +``depth`` Integer The depth of the event. +``origin`` String The name of the inviting homeserver. +``origin_server_ts`` Integer A timestamp added by the inviting + homeserver. +``prev_events`` List An event-reference list containing the + IDs of the immediate predecessor events. +``sender`` String The Matrix ID of the user who sent the + original ``m.room.third_party_invite``. +``state_key`` String The Matrix ID of the invited user. +``signatures`` Object The signature of the event from the + origin server. +``unsigned`` Object An object containing the properties that + aren't part of the signature's + computation. +======================== ============ ========================================= Where the ``content`` key contains the content for the ``m.room.member`` event specified in the `Client-Server API`_. Note that the ``membership`` property of @@ -1080,7 +1096,7 @@ and public keys the identity server provided as a response to the invite storage request. When a third-party identifier with pending invites gets bound to a Matrix ID, -the identity server will send a ``POST`` request to the ID's homeserver as described +the identity server will send a POST request to the ID's homeserver as described in the `Invitation Storage`_ section of the Identity Service API. The following process applies for each invite sent by the identity server: @@ -1102,13 +1118,13 @@ Where ``roomId`` is the ID of the room the invite is for. The required fields in the JSON body are: ==================== ======= ================================================== - Key Type Description + Key Type Description ==================== ======= ================================================== -``type`` String The event type. Must be `m.room.member`. +``type`` String The event type. Must be ``m.room.member``. ``room_id`` String The ID of the room the event is for. Must be the same as the ID specified in the path. ``sender`` String The Matrix ID of the user who sent the original - `m.room.third_party_invite`. + ``m.room.third_party_invite``. ``state_key`` String The Matrix ID of the invited user. ``content`` Object The content of the event. ==================== ======= ================================================== @@ -1121,7 +1137,7 @@ The inviting homeserver will then be able to authenticate the event. It will sen a fully authenticated event to the invited homeserver as described in the `Inviting to a room`_ section above. -Once the invited homeserver responded with the event to which it appended its +Once the invited homeserver responds with the event to which it appended its signature, the inviting homeserver will respond with ``200 OK`` and an empty body (``{}``) to the initial request on ``/exchange_third_party_invite/{roomId}`` and send the now verified ``m.room.member`` invite event to the room's members. @@ -1144,7 +1160,7 @@ It will then use these keys to verify that the ``signed`` object (in the ``third_party_invite`` object from the ``m.room.member`` event's content) was signed by the same identity server. -Since this ``signed`` object can only be delivered once in the ``POST`` request +Since this ``signed`` object can only be delivered once in the POST request emitted by the identity server upon binding between the third-party identifier and the Matrix ID, and contains the invited user's Matrix ID and the token delivered when the invite was stored, this verification will prove that the @@ -1161,10 +1177,10 @@ Every HTTP request made by a homeserver is authenticated using public key digital signatures. The request method, target and body are signed by wrapping them in a JSON object and signing it using the JSON signing algorithm. The resulting signatures are added as an Authorization header with an auth scheme -of X-Matrix. Note that the target field should include the full path starting with -``/_matrix/...``, including the ``?`` and any query parameters if present, but -should not include the leading ``https:``, nor the destination server's -hostname. +of ``X-Matrix``. Note that the target field should include the full path +starting with ``/_matrix/...``, including the ``?`` and any query parameters if +present, but should not include the leading ``https:``, nor the destination +server's hostname. Step 1 sign JSON: @@ -1360,7 +1376,7 @@ the following EDU:: messages: The messages to send. A map from user ID, to a map from device ID to message body. The device ID may also be *, meaning all known devices - for the user. + for the user Signing Events @@ -1492,7 +1508,7 @@ Servers can then transmit the entire event or the event with the non-essential keys removed. If the entire event is present, receiving servers can then check the event by computing the SHA-256 of the event, excluding the ``hash`` object. If the keys have been redacted, then the ``hash`` object is included when -calculating the SHA-256 instead. +calculating the SHA-256 hash instead. New hash functions can be introduced by adding additional keys to the ``hash`` object. Since the ``hash`` object cannot be redacted a server shouldn't allow From 54c3003b74a178d4c4455536baa84761c2f700ba Mon Sep 17 00:00:00 2001 From: user Date: Sun, 17 Jun 2018 16:35:23 +0100 Subject: [PATCH 081/308] newly joined -> newly-joined --- specification/server_server_api.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index e6484853..3738ea05 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -943,9 +943,9 @@ event to an resident homeserver, by using the ``send_join`` endpoint. This is invoked using the room ID and the event ID of the new member event. The resident homeserver then accepts this event into the room's event graph, -and responds to the joining server with the full set of state for the newly -joined room. This is returned as a two-element list, whose first element is the -integer 200, and whose second element is an object which contains the +and responds to the joining server with the full set of state for the +newly-joined room. This is returned as a two-element list, whose first element +is the integer 200, and whose second element is an object which contains the following keys: ======================== ============ ========================================= From fc402c296be202c7eda6ade54d4d0d29de6ce423 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 18 Jun 2018 12:03:14 -0600 Subject: [PATCH 082/308] Don't suggest reusing old issues for spec proposals In practice this was confusing for people, so instead we should encourage people to create new issues and reference the existing ones. Signed-off-by: Travis Ralston --- specification/proposals_intro.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index cfa59b8f..5cd8c554 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -30,9 +30,9 @@ The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: `_ is available. -- Make a new issue at https://github.com/matrix-org/matrix-doc/issues (or - modify an existing one), whose description should list the metadata as per - below. +- Make a new issue at https://github.com/matrix-org/matrix-doc/issues, whose + description should list the metadata as per below. If an existing issue + relates to your proposal, link to it in your new issue. - Gather feedback as widely as possible from the community and core team on the proposal. From d6471698066a348aee550aca68d6b3cbb183d008 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 18 Jun 2018 13:20:42 -0600 Subject: [PATCH 083/308] Add more strength to the suggestion to find existing issues Signed-off-by: Travis Ralston --- specification/proposals_intro.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specification/proposals_intro.rst b/specification/proposals_intro.rst index 5cd8c554..cc4d5d22 100644 --- a/specification/proposals_intro.rst +++ b/specification/proposals_intro.rst @@ -31,8 +31,9 @@ The process for submitting a Matrix Spec Change (MSC) Proposal is as follows: is available. - Make a new issue at https://github.com/matrix-org/matrix-doc/issues, whose - description should list the metadata as per below. If an existing issue - relates to your proposal, link to it in your new issue. + description should list the metadata as per below. Use the github search + function to attempt to locate any related github issues, and link any that + are found in the body of the new issue. - Gather feedback as widely as possible from the community and core team on the proposal. From 077cd04a6d42b852f0dd8e29ad26ffda843e48ef Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 20 Jun 2018 16:01:06 +0100 Subject: [PATCH 084/308] /limits => /config --- api/client-server/content-repo.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index cf7b5f25..3c200344 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -269,9 +269,9 @@ paths: "$ref": "definitions/error.yaml" tags: - Media - "/limits": + "/config": get: - summary: Get limits on what can be uploaded to the content repository. + summary: Get the config for the media repository. Clients SHOULD use this as a guide when uploading content. All values are intentionally left optional, the client MUST assume that any field not given is not limited. From 6b9640bcb499e95cb87ca6784b2b68e5bdf73d7a Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 20 Jun 2018 17:06:17 +0100 Subject: [PATCH 085/308] upload_size => m.upload.size --- api/client-server/content-repo.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 3c200344..b86b6596 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -290,12 +290,12 @@ paths: schema: type: object properties: - upload_size: + m.upload.size: type: number description: "The maximum size an upload can be in bytes." examples: application/json: { - "upload_size": 50000000 + "m.upload.size": 50000000 } tags: - Media From 6d0a56d190cddab7b0e710f6e1c43b04392088e0 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 22 Jun 2018 09:38:53 +0100 Subject: [PATCH 086/308] Suggestions by Travis tend to be sensible ones --- api/client-server/content-repo.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index b86b6596..4a61ccec 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -286,7 +286,7 @@ paths: - accessToken: [] responses: 200: - description: The limits exposed by the matrix server. + description: The public content repository configuration for the matrix server. schema: type: object properties: @@ -295,7 +295,7 @@ paths: description: "The maximum size an upload can be in bytes." examples: application/json: { - "m.upload.size": 50000000 - } + "m.upload.size": 50000000 + } tags: - Media From dfa4af5c19192967bdb3f7c49d5470f88b6c2760 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 22 Jun 2018 11:08:19 +0100 Subject: [PATCH 087/308] record that "identity server" has two words --- meta/documentation_style.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/meta/documentation_style.rst b/meta/documentation_style.rst index 698eb027..a67f25ff 100644 --- a/meta/documentation_style.rst +++ b/meta/documentation_style.rst @@ -63,7 +63,11 @@ have English as their first language. Prefer British English (colour, -ise) to American English. The word "homeserver" is spelt thus (rather than "home server", "Homeserver", -or (argh) "Home Server"). +or (argh) "Home Server"). However, an identity server is two words. + +.. Rationale: "homeserver" distinguishes from a "home server" which is a server + you have at home. "Identity server" is clear, whereas "identityserver" is + horrible. Lists should: From c84066df89dcbee70d90f3931850bc80631ca39a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 22 Jun 2018 14:30:24 +0100 Subject: [PATCH 088/308] do not use 'optional' in descriptions. --- meta/documentation_style.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/meta/documentation_style.rst b/meta/documentation_style.rst index a67f25ff..a866d1eb 100644 --- a/meta/documentation_style.rst +++ b/meta/documentation_style.rst @@ -100,3 +100,7 @@ When writing OpenAPI specifications for the API endpoints, follow these rules: The description is also the place to define default values for optional properties. Use the wording "Defaults to X [if unspecified]." + + Some descriptions start with the workd "Optional" to explicitly mark optional + properties and parameters. This is redundant. Instead, use the ``required`` + property to mark those that are required. From 2605c75f920973fafe4d57c29b89735dac9dfb53 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 22 Jun 2018 14:31:31 +0100 Subject: [PATCH 089/308] typo --- meta/documentation_style.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/documentation_style.rst b/meta/documentation_style.rst index a866d1eb..e60c7847 100644 --- a/meta/documentation_style.rst +++ b/meta/documentation_style.rst @@ -101,6 +101,6 @@ When writing OpenAPI specifications for the API endpoints, follow these rules: The description is also the place to define default values for optional properties. Use the wording "Defaults to X [if unspecified]." - Some descriptions start with the workd "Optional" to explicitly mark optional + Some descriptions start with the word "Optional" to explicitly mark optional properties and parameters. This is redundant. Instead, use the ``required`` property to mark those that are required. From 1140c0c05b73d081f421555465caa6bdb34a8c1e Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 22 Jun 2018 16:23:05 +0100 Subject: [PATCH 090/308] Document StateEvent for /createRoom (#1329) --- api/client-server/create_room.yaml | 4 ++++ changelogs/client_server.rst | 2 ++ 2 files changed, 6 insertions(+) diff --git a/api/client-server/create_room.yaml b/api/client-server/create_room.yaml index bf632c1a..3e88ee0c 100644 --- a/api/client-server/create_room.yaml +++ b/api/client-server/create_room.yaml @@ -150,10 +150,14 @@ paths: properties: type: type: string + description: The type of event to send. state_key: type: string + description: The state_key of the state event. Defaults to an empty string. content: type: object + description: The content of the event. + required: ["type", "content"] preset: type: string enum: ["private_chat", "public_chat", "trusted_private_chat"] diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 24344116..feabecab 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -37,6 +37,8 @@ Unreleased changes (`#1152 `_). - Mark ``GET /rooms/{roomId}/members`` as requiring authentication (`#1245 `_). + - Describe ``StateEvent`` for ``/createRoom`` + (`#1329 `_). - Changes to the API which will be backwards-compatible for clients: From 20200264e1ba2a31aa45b5ed67f38f1403577546 Mon Sep 17 00:00:00 2001 From: Remi Rampin Date: Sat, 23 Jun 2018 11:11:38 -0400 Subject: [PATCH 091/308] Consolidate backwards-compatible client changes --- changelogs/client_server.rst | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index feabecab..f375fc97 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -15,6 +15,22 @@ Unreleased changes - Sticker messages: - Add sticker message event definition. (`#1158 `_). + - Add 'token' parameter to /keys/query endpoint + (`#1104 `_). + - Add the room visibility options for the room directory + (`#1141 `_). + - Add spec for ignoring users + (`#1142 `_). + - Add the ``/register/available`` endpoint for username availability + (`#1151 `_). + - Add ``allow_remote`` to the content repo to avoid routing loops + (`#1265 `_). + - Add report content API + (`#1264 `_). + - Document ``/logout/all`` endpoint + (`#1263 `_). + - Document `highlights` field in /search response + (`#1274 `_). - Spec clarifications: @@ -40,25 +56,6 @@ Unreleased changes - Describe ``StateEvent`` for ``/createRoom`` (`#1329 `_). -- Changes to the API which will be backwards-compatible for clients: - - - Add 'token' parameter to /keys/query endpoint - (`#1104 `_). - - Add the room visibility options for the room directory - (`#1141 `_). - - Add spec for ignoring users - (`#1142 `_). - - Add the ``/register/available`` endpoint for username availability - (`#1151 `_). - - Add ``allow_remote`` to the content repo to avoid routing loops - (`#1265 `_). - - Add report content API - (`#1264 `_). - - Document ``/logout/all`` endpoint - (`#1263 `_). - - Document `highlights` field in /search response - (`#1274 `_). - r0.3.0 ====== From e52d94bea9cb7129d545d78e7d684926ddae8c67 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Mon, 25 Jun 2018 15:00:58 +0100 Subject: [PATCH 092/308] ignore malformed PR items --- scripts/proposals.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index b489f0d2..a8f58cae 100755 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -63,7 +63,6 @@ for label in labels: for item in issues[label]: # set the created date, find local field, otherwise Github - print(item) body = str(item['body']) created = re.search('^Date: (.+?)\n', body, flags=re.MULTILINE) if created is not None: @@ -133,16 +132,22 @@ for label in labels: text_file.write(" - " + str(shepherd) + "\n") # PRs - pr_list = re.search('PRs: (.+?)$', str(item['body'])) - if pr_list is not None: - pr_list_formatted = set() - pr_list = pr_list.group(1) - for p in pr_list.split(","): - prs.add(p.strip()) - pr_list_formatted.add("`PR" + str(p.strip()) + "`_") - text_file.write(" - " + ', '.join(pr_list_formatted)) - text_file.write("\n") - else: + try: + pr_list = re.search('PRs: (.+?)$', str(item['body'])) + if pr_list is not None: + pr_list_formatted = set() + pr_list = pr_list.group(1) + for p in pr_list.split(","): + if not re.match(r"#\d", p.strip()): + raise RuntimeWarning + prs.add(p.strip()) + pr_list_formatted.add("`PR" + str(p.strip()) + "`_") + text_file.write(" - " + ', '.join(pr_list_formatted)) + text_file.write("\n") + else: + text_file.write(" - \n") + except: + print("exception parsing PRs for MSC" + str(item['number'])) text_file.write(" - \n") text_file.write("\n\n\n") From f032d6d1c346e441cc666df73b528b8209e21011 Mon Sep 17 00:00:00 2001 From: Terrill Tsang Date: Tue, 26 Jun 2018 17:46:58 +0800 Subject: [PATCH 093/308] Update keys.yaml Fixed an end to end document bug , a format misleading in an claim response --- api/client-server/keys.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/keys.yaml b/api/client-server/keys.yaml index 5fb6318e..6e995c2c 100644 --- a/api/client-server/keys.yaml +++ b/api/client-server/keys.yaml @@ -274,7 +274,7 @@ paths: type: object description: |- One-time keys for the queried devices. A map from user ID, to a - map from ``:`` to the key object. + map from devices to a map from ``:`` to the key object. additionalProperties: type: object additionalProperties: From ddc152347bfa6c472ffc30d508b0a2fdbd3e8cb6 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 27 Jun 2018 20:14:29 +0100 Subject: [PATCH 094/308] Extending words and rate limiting --- api/client-server/content-repo.yaml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 4a61ccec..57585ce8 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -271,16 +271,17 @@ paths: - Media "/config": get: - summary: Get the config for the media repository. + summary: Get the configuration for the media repository. Clients SHOULD use this as a guide when uploading content. All values are intentionally left optional, the client MUST assume that any field not given is not limited. - **NOTE:** Reverse proxies may apply their own limits. + **NOTE:** Reverse proxies may apply their own configuration. + - If auth is not supplied, this endpoint gives the global limit of the server. - Otherwise it should give the limits applied to the authenticated user. - operationId: getLimits + If an accessToken is supplied, the configuration applied to the authenticated user. + Otherwise it should give the configuration applied globally to the server. + operationId: getConfig produces: ["application/json"] security: - accessToken: [] @@ -297,5 +298,10 @@ paths: application/json: { "m.upload.size": 50000000 } + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + tags: - Media From 80935eadcf87f6dd8e1a3669e5519844e77e34fb Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 28 Jun 2018 00:14:42 +0100 Subject: [PATCH 095/308] Update content-repo.yaml --- api/client-server/content-repo.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 57585ce8..c8096138 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -273,13 +273,13 @@ paths: get: summary: Get the configuration for the media repository. Clients SHOULD use this as a guide when uploading content. - All values are intentionally left optional, the client MUST assume - that any field not given is not limited. + All values are intentionally left optional. Clients SHOULD follow + the advise given in the field description when the field is not avaliable. **NOTE:** Reverse proxies may apply their own configuration. - If an accessToken is supplied, the configuration applied to the authenticated user. + If an accessToken is supplied, the configuration applied to the authenticated user is returned. Otherwise it should give the configuration applied globally to the server. operationId: getConfig produces: ["application/json"] @@ -293,7 +293,9 @@ paths: properties: m.upload.size: type: number - description: "The maximum size an upload can be in bytes." + description: |- + The maximum size an upload can be in bytes. If not listed or null, + the upload limit should be treated as unknown. examples: application/json: { "m.upload.size": 50000000 From eeaf4385d73f221e896917432ec6ed336bbabeaf Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 28 Jun 2018 09:04:01 +0100 Subject: [PATCH 096/308] Speling is my weakniss --- api/client-server/content-repo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index c8096138..be003a2b 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -274,7 +274,7 @@ paths: summary: Get the configuration for the media repository. Clients SHOULD use this as a guide when uploading content. All values are intentionally left optional. Clients SHOULD follow - the advise given in the field description when the field is not avaliable. + the advice given in the field description when the field is not available. **NOTE:** Reverse proxies may apply their own configuration. From 091b2a677190c1a120dd31c2ef4e4c0bec6469e6 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 28 Jun 2018 16:42:36 +0100 Subject: [PATCH 097/308] Hard limit on requiring accessToken. Rephrased some sections --- api/client-server/content-repo.yaml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index be003a2b..b48068f2 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -272,15 +272,13 @@ paths: "/config": get: summary: Get the configuration for the media repository. - Clients SHOULD use this as a guide when uploading content. + Clients SHOULD use this as a guide when using media endpoints. All values are intentionally left optional. Clients SHOULD follow the advice given in the field description when the field is not available. - **NOTE:** Reverse proxies may apply their own configuration. - - - If an accessToken is supplied, the configuration applied to the authenticated user is returned. - Otherwise it should give the configuration applied globally to the server. + **NOTE:** The /config endpoint is a guide. Other middleware such as + reverse proxies may apply their own configuration not described on this + endpoint. operationId: getConfig produces: ["application/json"] security: @@ -294,8 +292,9 @@ paths: m.upload.size: type: number description: |- - The maximum size an upload can be in bytes. If not listed or null, - the upload limit should be treated as unknown. + The maximum size an upload can be in bytes. + Clients SHOULD use this as a guide when uploading content. + If not listed or null, the size limit should be treated as unknown. examples: application/json: { "m.upload.size": 50000000 From 1820df02d9faebf042bb3cf81e24d2036e844f74 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 28 Jun 2018 17:07:44 +0100 Subject: [PATCH 098/308] media => content repository --- api/client-server/content-repo.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index b48068f2..582d0695 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -271,8 +271,8 @@ paths: - Media "/config": get: - summary: Get the configuration for the media repository. - Clients SHOULD use this as a guide when using media endpoints. + summary: Get the configuration for the content repository. + Clients SHOULD use this as a guide when using content endpoints. All values are intentionally left optional. Clients SHOULD follow the advice given in the field description when the field is not available. From 21e957edfba242107eb871a960fc1af334b8ce49 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 28 Jun 2018 17:23:57 +0100 Subject: [PATCH 099/308] Better note for /config --- api/client-server/content-repo.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 582d0695..8ccc7684 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -276,9 +276,11 @@ paths: All values are intentionally left optional. Clients SHOULD follow the advice given in the field description when the field is not available. - **NOTE:** The /config endpoint is a guide. Other middleware such as - reverse proxies may apply their own configuration not described on this - endpoint. + **NOTE:** Both clients and server administrators should be aware that proxies + between the client and the server may affect the apparent behaviour of content + repository APIs, for example, proxies may enforce a lower upload size limit + than is advertised by the server on this endpoint. Clients should handle such + situations gracefully. operationId: getConfig produces: ["application/json"] security: From 41c18bed0f954e43e65993b7487a573bf65152f0 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 28 Jun 2018 17:53:57 +0100 Subject: [PATCH 100/308] +repository --- api/client-server/content-repo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 8ccc7684..e2549c65 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -272,7 +272,7 @@ paths: "/config": get: summary: Get the configuration for the content repository. - Clients SHOULD use this as a guide when using content endpoints. + Clients SHOULD use this as a guide when using content repository endpoints. All values are intentionally left optional. Clients SHOULD follow the advice given in the field description when the field is not available. From a1309d636c8af90ef04499c392ce836fb07d614f Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 29 Jun 2018 09:58:07 +0100 Subject: [PATCH 101/308] Remove "Clients should handle gracefully" sentence --- api/client-server/content-repo.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index e2549c65..0f4ef2da 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -279,8 +279,7 @@ paths: **NOTE:** Both clients and server administrators should be aware that proxies between the client and the server may affect the apparent behaviour of content repository APIs, for example, proxies may enforce a lower upload size limit - than is advertised by the server on this endpoint. Clients should handle such - situations gracefully. + than is advertised by the server on this endpoint. operationId: getConfig produces: ["application/json"] security: From d73b247688606d767667c5e80a1b35c69513c566 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Jun 2018 10:57:35 +0100 Subject: [PATCH 102/308] 3PE lookup service --- .../application_service.yaml | 391 +++++++++++++++++- changelogs/client_server.rst | 12 + specification/targets.yaml | 1 + 3 files changed, 401 insertions(+), 3 deletions(-) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index c39ce198..a0713d45 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -1,4 +1,5 @@ # Copyright 2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -91,7 +92,13 @@ paths: } schema: type: object - + 500: + description: There was a problem completing the request. + examples: + application/json: { + } + schema: + type: object "/rooms/{roomAlias}": get: summary: Query if a room alias should exist on the application service. @@ -147,7 +154,14 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + "errcode": "M_NOT_FOUND" + } + schema: + type: object + 500: + description: There was a problem completing the request. + examples: + application/json: { } schema: type: object @@ -203,7 +217,378 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + "errcode": "M_NOT_FOUND" } schema: type: object + 500: + description: There was a problem completing the request. + examples: + application/json: { + } + schema: + type: object + "/_matrix/app/unstable/thirdparty/{protocol}": + get: + summary: Retrieve metadata about a specific protocol that the application service supports. + description: |- + This API is called by the HS when it wants to present clients with + specific information about the various thirdparty networks that an AS + supports. + operationId: queryMetadata + parameters: + - in: path + name: protocol + type: string + description: |- + The name of the protocol. + required: true + x-example: "irc" + responses: + 200: + description: The protocol was found and metadata returned. + examples: + application/json: { + "user_fields": ["network", "nickname"], + "location_fields": ["network", "channel"], + "icon": "mxc://example.org/aBcDeFgH", + "field_types": { + "network": { + "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", + "placeholder": "irc.example.org" + }, + "nickname": { + "regexp": "[^\\s]+", + "placeholder": "username" + }, + "channel": { + "regexp": "#[^\\s]+", + "placeholder": "#foobar" + } + }, + "instances": [ + { + "desc": "Freenode", + "icon": "mxc://example.org/JkLmNoPq", + "fields": { + "network": "freenode.net", + } + } + ] + } + schema: + type: object + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + type: object + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: { + "errcode": "M_FORBIDDEN" + } + schema: + type: object + 404: + description: No protocol was found with the given path. + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object + 500: + description: There was a problem completing the request. + examples: + application/json: { + } + schema: + type: object + "/_matrix/app/unstable/thirdparty/user/{protocol}": + get: + summary: Retrieve the Matrix ID of a corresponding thirdparty user. + description: |- + This API is called by the HS in order to retrieve a Matrix ID linked + to a user on the external service, given a set of user parameters. + operationId: queryUserByProtocol + parameters: + - in: path + name: protocol + type: string + description: |- + The name of the protocol. + required: true + x-example: irc + - in: query + name: field1, field2... + type: string + description: |- + One or more custom fields that are passed to the AS to help identify the user. + responses: + 200: + description: The Matrix IDs found with the given parameters. + examples: + application/json: [{ + "userid": "@_gitter_jim:matrix.org", + "protocol": "gitter", + "fields": { + "user": "jim" + } + }] + schema: + type: array + description: Matched users. + items: + type: object + title: User + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + type: object + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: { + "errcode": "M_FORBIDDEN" + } + schema: + type: object + 404: + description: No users were found with the given parameters. + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object + 500: + description: There was a problem completing the request. + examples: + application/json: { + } + schema: + type: object + "/_matrix/app/unstable/thirdparty/location/{protocol}": + get: + summary: Retreive Matrix-side portals rooms leading to a thirdparty location. + description: |- + Requesting this endpoint with a valid protocol name results in a list + of successful mapping results in a JSON array. Each result contains + objects to represent the Matrix room or rooms that represent a portal + to this 3PN. Each has the Matrix room alias string, an identifier for + the particular 3PN protocol, and an object containing the + network-specific fields that comprise this identifier. It should + attempt to canonicalise the identifier as much as reasonably possible + given the network type. + operationId: queryLocationByProtocol + parameters: + - in: path + name: protocol + type: string + description: The protocol used to communicate to the thirdparty network. + required: true + x-example: "irc" + - in: query + name: field1, field2... + type: string + description: |- + One or more custom fields that are passed to the AS to help + identify the thirdparty location. + responses: + 200: + description: At least one portal room was found. + examples: + application/json: [{ + "alias": "#freenode_#matrix:matrix.org", + "protocol": "irc", + "fields": { + "network": "freenode", + "channel": "#matrix" + } + }] + schema: + type: array + description: |- + Array of portal rooms leading to the requested thirdparty + location. + items: + type: object + title: Portal Room + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + type: object + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: { + "errcode": "M_FORBIDDEN" + } + schema: + type: object + 404: + description: No mappings were found with the given parameters. + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object + 500: + description: There was a problem completing the request. + examples: + application/json: { + } + schema: + type: object + "/_matrix/app/unstable/thirdparty/location": + get: + summary: Reverse-lookup thirdparty locations given a Matrix room alias. + description: |- + Retreive an array of thirdparty network locations from a Matrix room + alias. + operationId: queryLocationByAlias + parameters: + - in: path + name: alias + type: string + description: The Matrix room alias to look up. + responses: + 200: + description: |- + All found thirdparty locations. Same response format as the + forward lookup response. + examples: + application/json: [{ + "alias": "#freenode_#matrix:matrix.org", + "protocol": "irc", + "fields": { + "network": "freenode", + "channel": "#matrix" + } + }] + schema: + type: array + description: Matched thirdparty locations. + items: + type: object + title: Location + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + type: object + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: { + "errcode": "M_FORBIDDEN" + } + schema: + type: object + 404: + description: No mappings were found with the given parameters. + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object + 500: + description: There was a problem completing the request. + examples: + application/json: { + } + schema: + type: object + "/_matrix/app/unstable/thirdparty/user": + get: + summary: Reverse-lookup thirdparty users given a Matrix ID. + description: |- + Retreive an array of thirdparty users from a Matrix ID. + operationId: queryUserByID + paramters: + - in: path + name: userid + type: string + description: The Matrix ID to look up. + responses: + 200: + description: |- + An array of thirdparty users. + examples: + application/json: [{ + "userid": "@_gitter_jim:matrix.org", + "protocol": "gitter", + "fields": { + "user": "jim" + } + }] + schema: + type: array + description: Matched thirdparty users + items: + type: object + title: User + 401: + description: |- + The homeserver has not supplied credentials to the application service. + Optional error information can be included in the body of this response. + examples: + application/json: { + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } + schema: + type: object + 403: + description: |- + The credentials supplied by the homeserver were rejected. + examples: + application/json: { + "errcode": "M_FORBIDDEN" + } + schema: + type: object + 404: + description: No mappings were found with the given parameters. + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object + 500: + description: There was a problem completing the request. + examples: + application/json: { + } + schema: + type: object \ No newline at end of file diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index feabecab..f7e94dbe 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -11,6 +11,18 @@ Unreleased changes (`#1110 `_). - ``POST /delete_devices`` (`#1239 `_). + - ``GET /thirdparty/protocols`` + (`#1353 `_). + - ``GET /thirdparty/protocol/{protocol}`` + (`#1353 `_). + - ``GET /thirdparty/location/{protocol}`` + (`#1353 `_). + - ``GET /thirdparty/user/{protocol}`` + (`#1353 `_). + - ``GET /thirdparty/location`` + (`#1353 `_). + - ``GET /thirdparty/user`` + (`#1353 `_). - Sticker messages: - Add sticker message event definition. diff --git a/specification/targets.yaml b/specification/targets.yaml index bc9d94c8..c6c9cd23 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -12,6 +12,7 @@ targets: - { 1: modules.rst } - { 2: feature_profiles.rst } - { 2: "group:modules" } # reference a group of files + - { 1: thirdparty_lookup.rst } version_label: "%CLIENT_RELEASE_LABEL%" application_service: files: From 5803c9cace238c6ce72bb52c0800e064ec5d6d48 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Tue, 3 Jul 2018 15:49:40 +0100 Subject: [PATCH 103/308] don't run proposals.py step if in TravisCI environment --- scripts/generate-matrix-org-assets | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/generate-matrix-org-assets b/scripts/generate-matrix-org-assets index ed08f81d..cbebbb64 100755 --- a/scripts/generate-matrix-org-assets +++ b/scripts/generate-matrix-org-assets @@ -8,8 +8,11 @@ cd `dirname $0`/.. mkdir -p assets -# generate specification/proposals.rst -./scripts/proposals.py +if [ "$TRAVIS" != "true" ] +then + # generate specification/proposals.rst + ./scripts/proposals.py +fi # generate the spec docs ./scripts/gendoc.py -d assets/spec From eb21a71b702e481680800faba0dd3aa354df8c54 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 10:49:55 -0600 Subject: [PATCH 104/308] Make the builder happier with Windows environments There's two main issues with Windows environments: * Slashes * Windows still notifies about some directories we've ignored, therefore we need to filter them too It's not super pretty, but it does work. --- scripts/continuserv/main.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index 330ddca7..fe4e5cc9 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -99,6 +99,7 @@ func makeWalker(base string, w *fsnotify.Watcher) filepath.WalkFunc { } // skip a few things that we know don't form part of the spec + rel = strings.Replace(rel, "\\", "/", -1) // normalize slashes (thanks to windows) if rel == "api/node_modules" || rel == "scripts/gen" || rel == "scripts/tmp" { @@ -125,6 +126,14 @@ func filter(e fsnotify.Event) bool { return false } + // Forcefully ignore directories we don't care about (Windows, at least, tries to notify about some directories) + filePath := strings.Replace(e.Name, "\\", "/", -1) // normalize slashes (thanks to windows) + if strings.Contains(filePath, "/scripts/tmp") || + strings.Contains(filePath, "/scripts/gen") || + strings.Contains(filePath, "/api/node_modules") { + return false + } + return true } @@ -149,6 +158,10 @@ func serve(w http.ResponseWriter, req *http.Request) { } b, ok = m.bytes[file] + if !ok { + b, ok = m.bytes[strings.Replace(file, "/", "\\", -1)] // Attempt a Windows lookup + } + if ok && file == "api-docs.json" { w.Header().Set("Access-Control-Allow-Origin", "*") } From 85e1962f9ceabd64d0a72d20eb651fe50172670c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 12:07:31 -0600 Subject: [PATCH 105/308] Use filepath.To/FromSlash instead of manually replacing slashes see https://github.com/matrix-org/matrix-doc/pull/1356#discussion_r199897557 --- scripts/continuserv/main.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index fe4e5cc9..3797b9d7 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -98,8 +98,10 @@ func makeWalker(base string, w *fsnotify.Watcher) filepath.WalkFunc { log.Fatalf("Failed to get relative path of %s: %v", path, err) } + // Normalize slashes + rel = filepath.ToSlash(rel) + // skip a few things that we know don't form part of the spec - rel = strings.Replace(rel, "\\", "/", -1) // normalize slashes (thanks to windows) if rel == "api/node_modules" || rel == "scripts/gen" || rel == "scripts/tmp" { @@ -127,7 +129,7 @@ func filter(e fsnotify.Event) bool { } // Forcefully ignore directories we don't care about (Windows, at least, tries to notify about some directories) - filePath := strings.Replace(e.Name, "\\", "/", -1) // normalize slashes (thanks to windows) + filePath := filepath.ToSlash(e.Name) // normalize slashes if strings.Contains(filePath, "/scripts/tmp") || strings.Contains(filePath, "/scripts/gen") || strings.Contains(filePath, "/api/node_modules") { @@ -156,11 +158,7 @@ func serve(w http.ResponseWriter, req *http.Request) { if file[0] == '/' { file = file[1:] } - b, ok = m.bytes[file] - - if !ok { - b, ok = m.bytes[strings.Replace(file, "/", "\\", -1)] // Attempt a Windows lookup - } + b, ok = m.bytes[filepath.FromSlash(file)] // de-normalize slashes if ok && file == "api-docs.json" { w.Header().Set("Access-Control-Allow-Origin", "*") From a2e0d0382ca46a7564581fed0e3fbfc4093b08fb Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Tue, 3 Jul 2018 14:10:22 -0400 Subject: [PATCH 106/308] ignore emacs temporary files and fix ignoring of dotfiles --- scripts/continuserv/main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/continuserv/main.go b/scripts/continuserv/main.go index 330ddca7..21effdfd 100644 --- a/scripts/continuserv/main.go +++ b/scripts/continuserv/main.go @@ -120,8 +120,11 @@ func filter(e fsnotify.Event) bool { return false } - // Avoid some temp files that vim writes - if strings.HasSuffix(e.Name, "~") || strings.HasSuffix(e.Name, ".swp") || strings.HasPrefix(e.Name, ".") { + _, fname := filepath.Split(e.Name) + + // Avoid some temp files that vim or emacs writes + if strings.HasSuffix(e.Name, "~") || strings.HasSuffix(e.Name, ".swp") || strings.HasPrefix(fname, ".") || + (strings.HasPrefix(fname, "#") && strings.HasSuffix(fname, "#")) { return false } From aac7b848f77e4aeb5f72f7858ea4027f01d2b3c0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 13:10:28 -0600 Subject: [PATCH 107/308] Add uhoreg and myself as trusted speculator people --- scripts/speculator/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 0380a3cd..5e122503 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -713,6 +713,8 @@ func main() { "ara4n": true, "leonerd": true, "rxl881": true, + "uhoreg": true, + "turt2live": true, } if err := initCache(); err != nil { log.Fatal(err) From 2409c849686cf7028dd35d5c12cfb6fb67ed5565 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 13:41:49 -0600 Subject: [PATCH 108/308] Document the GET version of /login Fixes https://github.com/matrix-org/matrix-doc/issues/677 --- api/client-server/login.yaml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/api/client-server/login.yaml b/api/client-server/login.yaml index a6e21a38..c0db2299 100644 --- a/api/client-server/login.yaml +++ b/api/client-server/login.yaml @@ -28,6 +28,42 @@ securityDefinitions: $ref: definitions/security.yaml paths: "/login": + get: + summary: Get the supported login types to authenticate users + description: |- + Gets the homeserver's supported login types to authenticate users. Clients + should pick one of these and supply it as the ``type`` when logging in. + operationId: getLoginFlows + responses: + 200: + description: The login types the homeserver supports + examples: + application/json: { + "flows": [ + {"type": "m.login.password"} + ] + } + schema: + type: object + properties: + flows: + type: array + description: The homeserver's supported login types + items: + type: object + title: LoginFlow + properties: + type: + description: |- + The login type. This is supplied as the ``type`` when + logging in. + type: string + 429: + description: This request was rate-limited. + schema: + "$ref": "definitions/error.yaml" + tags: + - Session management post: summary: Authenticates the user. description: |- From 17c9cde0ec06e2f85b4e08615041890c4c223218 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 13:46:57 -0600 Subject: [PATCH 109/308] Add to changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index feabecab..44d41f4e 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -15,6 +15,8 @@ Unreleased changes - Sticker messages: - Add sticker message event definition. (`#1158 `_). + - Document the GET version of ``/login`` + (`#1361 `_). - Spec clarifications: From 7d94aaace8a799927673d53fa49b5f28c7d83acd Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 14:59:41 -0600 Subject: [PATCH 110/308] Explicitly declare that the kick/ban reason will be on the membership event Fixes https://github.com/matrix-org/matrix-doc/issues/798 --- api/client-server/banning.yaml | 3 ++- api/client-server/kicking.yaml | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/api/client-server/banning.yaml b/api/client-server/banning.yaml index 3877f0e9..4030f46b 100644 --- a/api/client-server/banning.yaml +++ b/api/client-server/banning.yaml @@ -61,7 +61,8 @@ paths: description: The fully qualified user ID of the user being banned. reason: type: string - description: The reason the user has been banned. + description: The reason the user has been banned. This will be supplied as the + ``reason`` on the target's updated `m.room.member`_ event. required: ["user_id"] responses: 200: diff --git a/api/client-server/kicking.yaml b/api/client-server/kicking.yaml index ef43c0ee..bacc806e 100644 --- a/api/client-server/kicking.yaml +++ b/api/client-server/kicking.yaml @@ -34,6 +34,10 @@ paths: Kick a user from the room. The caller must have the required power level in order to perform this operation. + + Kicking a user adjusts the target member's membership state to be ``leave`` with an + optional ``reason``. Like with other membership changes, a user can directly adjust + the target member's state by making a request to ``/rooms//state/m.room.member/``. operationId: kick security: - accessToken: [] @@ -59,7 +63,9 @@ paths: description: The fully qualified user ID of the user being kicked. reason: type: string - description: The reason the user has been kicked. + description: |- + The reason the user has been kicked. This will be supplied as the + ``reason`` on the target's updated `m.room.member`_ event. required: ["user_id"] responses: 200: From 5b6ad6f39d5792c987b4bee60989b8a2b026f9b3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 15:19:33 -0600 Subject: [PATCH 111/308] Add to changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index feabecab..a16b96ae 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -39,6 +39,8 @@ Unreleased changes (`#1245 `_). - Describe ``StateEvent`` for ``/createRoom`` (`#1329 `_). + - Describe how the ``reason`` is handled for kicks/bans + (`#1362 `_). - Changes to the API which will be backwards-compatible for clients: From d279fdafa6e649f4dd8ea2dd4a89e559c633a3f3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 15:49:16 -0600 Subject: [PATCH 112/308] Document the server_name query parameter on /join/{roomIdOrAlias} Fixes https://github.com/matrix-org/matrix-doc/issues/904 --- api/client-server/joining.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/api/client-server/joining.yaml b/api/client-server/joining.yaml index da4e4337..bd3e6885 100644 --- a/api/client-server/joining.yaml +++ b/api/client-server/joining.yaml @@ -143,6 +143,16 @@ paths: description: The room identifier or alias to join. required: true x-example: "#monkeys:matrix.org" + - in: query + type: array + items: + type: string + explode: true + name: server_name + description: |- + The servers to attempt to join the room through. One of the servers + must be participating in the room. + x-example: ["matrix.org", "elsewhere.ca"] - in: body name: third_party_signed schema: From cbbdcbcf01cd5c9d18bb0ca0aa80257dad8fccfe Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 15:50:01 -0600 Subject: [PATCH 113/308] Show arrays in the query string as "[type]" rather than "array" Otherwise the question "array of what?" is raised. --- scripts/templating/matrix_templates/units.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index d8cfe1e3..dac183d1 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -502,6 +502,11 @@ class MatrixUnits(Units): # assign value expected for this param val_type = param.get("type") # integer/string + if val_type == "array": + items = param.get("items") + if items: + val_type = "[%s]" % items.get("type") + if param.get("enum"): val_type = "enum" desc += ( From 625913d797cda10d9916934351e598161adaced0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 15:53:43 -0600 Subject: [PATCH 114/308] Add to changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index feabecab..01670c74 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -15,6 +15,8 @@ Unreleased changes - Sticker messages: - Add sticker message event definition. (`#1158 `_). + - Document the ``server_name`` parameter on ``/join/{roomIdOrAlias}`` + (`#1364 `_). - Spec clarifications: From 4ca54404fa4ae3b2ae58c34688204883e4c94903 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 16:15:29 -0600 Subject: [PATCH 115/308] Document the CORS/preflight headers Fixes https://github.com/matrix-org/matrix-doc/issues/1006 --- specification/client_server_api.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index dec3a4f4..f7a7d509 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -164,6 +164,26 @@ recommended. {{versions_cs_http_api}} +Web Browser Clients +------------------- + +It is realistic to expect that some clients will be written to be run within a +web browser or similar environment. In these cases, the homeserver should respond +to pre-flight requests and supply Cross-Origin Resource Sharing (CORS) headers. + +When a client approaches the server with a pre-flight (``OPTIONS``) request, the +server should respond with the CORS headers for that route. If the route does not +exist, the server should return an ``M_NOT_FOUND`` error with a 404 status code. + +The standard CORS headers to be returned by servers on all requests are: + +.. code:: + + Access-Control-Allow-Origin: * + Access-Control-Allow-Methods: GET, POST, PUT, DELETE, OPTIONS + Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept, Authorization + + Client Authentication --------------------- From b8c8d859268874cd548aff3be9fe58eed50a15a6 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 16:18:04 -0600 Subject: [PATCH 116/308] Add to changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index feabecab..e4e7552f 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -15,6 +15,8 @@ Unreleased changes - Sticker messages: - Add sticker message event definition. (`#1158 `_). + - Document the CORS/preflight headers + (`#1365 `_). - Spec clarifications: From a6b0125da62e9c911ff15b8cbcf267499f95ee99 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 3 Jul 2018 16:19:31 -0600 Subject: [PATCH 117/308] Our swagger doesn't support explode yet --- api/client-server/joining.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/api/client-server/joining.yaml b/api/client-server/joining.yaml index bd3e6885..471c5038 100644 --- a/api/client-server/joining.yaml +++ b/api/client-server/joining.yaml @@ -147,7 +147,6 @@ paths: type: array items: type: string - explode: true name: server_name description: |- The servers to attempt to join the room through. One of the servers From 2fc292646147b236a3740184a589f457206b3db5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 11:33:51 -0600 Subject: [PATCH 118/308] Clearly say that the server/server API is unstable and subject to change --- specification/server_server_api.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 3738ea05..f2b3204a 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -16,6 +16,10 @@ Federation API ============== +.. WARNING:: + This API is unstable and will change without warning or discussion while + we work towards a r0 release (scheduled for August 2018). + Matrix homeservers use the Federation APIs (also known as server-server APIs) to communicate with each other. Homeservers use these APIs to push messages to each other in real-time, to From 1f12637d3f8973301e0ec6b7ebdfb167603b7d49 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 12:23:51 -0600 Subject: [PATCH 119/308] Getting presence requires authentication --- api/client-server/presence.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/client-server/presence.yaml b/api/client-server/presence.yaml index ba202c2d..ab32f983 100644 --- a/api/client-server/presence.yaml +++ b/api/client-server/presence.yaml @@ -83,6 +83,8 @@ paths: description: |- Get the given user's presence state. operationId: getPresence + security: + - accessToken: [] parameters: - in: path type: string From b61fe201718ca121c62a8edde9093bb2c0611bf7 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 13:02:32 -0600 Subject: [PATCH 120/308] Add to changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 44d41f4e..eb0387c1 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -39,6 +39,8 @@ Unreleased changes (`#1152 `_). - Mark ``GET /rooms/{roomId}/members`` as requiring authentication (`#1245 `_). + - Mark ``GET /presence/{userId}/status`` as requiring authentication + (`#1371 `_). - Describe ``StateEvent`` for ``/createRoom`` (`#1329 `_). From 7cd22f5690b50cc4132328e3ba030b49f117e83c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 13:45:34 -0600 Subject: [PATCH 121/308] Describe the rate limit error everywhere Fixes https://github.com/matrix-org/matrix-doc/issues/1153 --- api/client-server/content-repo.yaml | 10 +++---- .../definitions/{ => errors}/error.yaml | 2 ++ .../definitions/errors/rate_limited.yaml | 26 +++++++++++++++++++ api/client-server/inviting.yaml | 2 +- api/client-server/joining.yaml | 4 +-- api/client-server/leaving.yaml | 4 +-- api/client-server/login.yaml | 4 +-- api/client-server/presence.yaml | 4 +-- api/client-server/profile.yaml | 4 +-- api/client-server/pusher.yaml | 2 +- api/client-server/pushrules.yaml | 2 +- api/client-server/receipts.yaml | 2 +- api/client-server/registration.yaml | 8 +++--- api/client-server/search.yaml | 2 +- api/client-server/third_party_membership.yaml | 2 +- api/client-server/typing.yaml | 2 +- api/client-server/users.yaml | 2 +- api/client-server/voip.yaml | 2 +- api/client-server/whoami.yaml | 2 +- 19 files changed, 57 insertions(+), 29 deletions(-) rename api/client-server/definitions/{ => errors}/error.yaml (92%) create mode 100644 api/client-server/definitions/errors/rate_limited.yaml diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index f76b2fb9..7ded75d0 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -72,7 +72,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Media "/download/{serverName}/{mediaId}": @@ -119,7 +119,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Media "/download/{serverName}/{mediaId}/{fileName}": @@ -173,7 +173,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Media "/thumbnail/{serverName}/{mediaId}": @@ -238,7 +238,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Media "/preview_url": @@ -293,6 +293,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Media diff --git a/api/client-server/definitions/error.yaml b/api/client-server/definitions/errors/error.yaml similarity index 92% rename from api/client-server/definitions/error.yaml rename to api/client-server/definitions/errors/error.yaml index fa5cada7..7471da6f 100644 --- a/api/client-server/definitions/error.yaml +++ b/api/client-server/definitions/errors/error.yaml @@ -17,7 +17,9 @@ properties: errcode: type: string description: An error code. + example: M_UNKNOWN error: type: string description: A human-readable error message. + example: An unknown error occurred required: ["errcode"] \ No newline at end of file diff --git a/api/client-server/definitions/errors/rate_limited.yaml b/api/client-server/definitions/errors/rate_limited.yaml new file mode 100644 index 00000000..6b4932ee --- /dev/null +++ b/api/client-server/definitions/errors/rate_limited.yaml @@ -0,0 +1,26 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +$ref: error.yaml +type: object +description: The rate limit was reached for this request +properties: + errcode: + type: string + description: The M_LIMIT_EXCEEDED error code + example: M_LIMIT_EXCEEDED + error: + type: string + description: A human-readable error message. + example: Too many requests +required: ["errcode"] \ No newline at end of file diff --git a/api/client-server/inviting.yaml b/api/client-server/inviting.yaml index 47f51bd4..1e6694dc 100644 --- a/api/client-server/inviting.yaml +++ b/api/client-server/inviting.yaml @@ -96,6 +96,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Room membership diff --git a/api/client-server/joining.yaml b/api/client-server/joining.yaml index da4e4337..9890e73c 100644 --- a/api/client-server/joining.yaml +++ b/api/client-server/joining.yaml @@ -113,7 +113,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Room membership "/join/{roomIdOrAlias}": @@ -209,6 +209,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Room membership diff --git a/api/client-server/leaving.yaml b/api/client-server/leaving.yaml index 36351fd4..1a2515a6 100644 --- a/api/client-server/leaving.yaml +++ b/api/client-server/leaving.yaml @@ -64,7 +64,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Room membership "/rooms/{roomId}/forget": @@ -102,6 +102,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Room membership diff --git a/api/client-server/login.yaml b/api/client-server/login.yaml index c0db2299..e64ff953 100644 --- a/api/client-server/login.yaml +++ b/api/client-server/login.yaml @@ -61,7 +61,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Session management post: @@ -176,6 +176,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Session management diff --git a/api/client-server/presence.yaml b/api/client-server/presence.yaml index ba202c2d..2a97e1ba 100644 --- a/api/client-server/presence.yaml +++ b/api/client-server/presence.yaml @@ -75,7 +75,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Presence get: @@ -176,7 +176,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Presence get: diff --git a/api/client-server/profile.yaml b/api/client-server/profile.yaml index 0cada0ca..c8dc4056 100644 --- a/api/client-server/profile.yaml +++ b/api/client-server/profile.yaml @@ -67,7 +67,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - User data get: @@ -141,7 +141,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - User data get: diff --git a/api/client-server/pusher.yaml b/api/client-server/pusher.yaml index 9cf40a06..826b69d1 100644 --- a/api/client-server/pusher.yaml +++ b/api/client-server/pusher.yaml @@ -232,6 +232,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Push notifications diff --git a/api/client-server/pushrules.yaml b/api/client-server/pushrules.yaml index 801349ef..22738c3e 100644 --- a/api/client-server/pushrules.yaml +++ b/api/client-server/pushrules.yaml @@ -442,7 +442,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Push notifications "/pushrules/{scope}/{kind}/{ruleId}/enabled": diff --git a/api/client-server/receipts.yaml b/api/client-server/receipts.yaml index e46359a9..a3e9789e 100644 --- a/api/client-server/receipts.yaml +++ b/api/client-server/receipts.yaml @@ -76,6 +76,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Room participation diff --git a/api/client-server/registration.yaml b/api/client-server/registration.yaml index 1c544afe..5c205999 100644 --- a/api/client-server/registration.yaml +++ b/api/client-server/registration.yaml @@ -185,7 +185,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - User data "/register/email/requestToken": @@ -296,7 +296,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - User data "/account/password/email/requestToken": @@ -363,7 +363,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - User data "/register/available": @@ -423,6 +423,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - User data diff --git a/api/client-server/search.yaml b/api/client-server/search.yaml index e5f6a872..3dcd719f 100644 --- a/api/client-server/search.yaml +++ b/api/client-server/search.yaml @@ -355,6 +355,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Search diff --git a/api/client-server/third_party_membership.yaml b/api/client-server/third_party_membership.yaml index 612b22d0..3050ee3a 100644 --- a/api/client-server/third_party_membership.yaml +++ b/api/client-server/third_party_membership.yaml @@ -129,6 +129,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Room membership diff --git a/api/client-server/typing.yaml b/api/client-server/typing.yaml index e2a8f9bd..e7cbe2d7 100644 --- a/api/client-server/typing.yaml +++ b/api/client-server/typing.yaml @@ -82,6 +82,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - Room participation diff --git a/api/client-server/users.yaml b/api/client-server/users.yaml index 1734e3bb..a682b435 100644 --- a/api/client-server/users.yaml +++ b/api/client-server/users.yaml @@ -95,6 +95,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - User data diff --git a/api/client-server/voip.yaml b/api/client-server/voip.yaml index 6d6136a5..75ace4c3 100644 --- a/api/client-server/voip.yaml +++ b/api/client-server/voip.yaml @@ -73,6 +73,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - VOIP diff --git a/api/client-server/whoami.yaml b/api/client-server/whoami.yaml index 8f5abdf7..4d178e07 100644 --- a/api/client-server/whoami.yaml +++ b/api/client-server/whoami.yaml @@ -79,6 +79,6 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/rate_limited.yaml" tags: - User data From caa6d4d9ad082003cb4c481795327c5cc22569c3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 13:51:17 -0600 Subject: [PATCH 122/308] Give all errors a schema reference This just helps keep an overall structure --- api/client-server/administrative_contact.yaml | 2 ++ api/client-server/banning.yaml | 4 ++++ api/client-server/create_room.yaml | 2 ++ api/client-server/directory.yaml | 4 ++++ api/client-server/inviting.yaml | 2 ++ api/client-server/joining.yaml | 4 ++++ api/client-server/kicking.yaml | 2 ++ api/client-server/list_public_rooms.yaml | 4 ++++ api/client-server/login.yaml | 4 ++++ api/client-server/pusher.yaml | 2 +- api/client-server/pushrules.yaml | 2 +- api/client-server/registration.yaml | 6 +++++- api/client-server/third_party_membership.yaml | 2 ++ api/client-server/whoami.yaml | 4 ++-- 14 files changed, 39 insertions(+), 5 deletions(-) diff --git a/api/client-server/administrative_contact.yaml b/api/client-server/administrative_contact.yaml index e7381a55..8f0319d5 100644 --- a/api/client-server/administrative_contact.yaml +++ b/api/client-server/administrative_contact.yaml @@ -129,6 +129,8 @@ paths: "errcode": "M_THREEPID_AUTH_FAILED", "error": "The third party credentials could not be verified by the identity server." } + schema: + "$ref": "definitions/errors/error.yaml" tags: - User data "/account/3pid/email/requestToken": diff --git a/api/client-server/banning.yaml b/api/client-server/banning.yaml index 3877f0e9..9aceb2db 100644 --- a/api/client-server/banning.yaml +++ b/api/client-server/banning.yaml @@ -82,6 +82,8 @@ paths: "errcode": "M_FORBIDDEN", "error": "You do not have a high enough power level to ban from this room." } + schema: + "$ref": "definitions/errors/error.yaml" tags: - Room membership "/rooms/{roomId}/unban": @@ -133,5 +135,7 @@ paths: "errcode": "M_FORBIDDEN", "error": "You do not have a high enough power level to unban from this room." } + schema: + "$ref": "definitions/errors/error.yaml" tags: - Room membership diff --git a/api/client-server/create_room.yaml b/api/client-server/create_room.yaml index 3e88ee0c..59f7f75f 100644 --- a/api/client-server/create_room.yaml +++ b/api/client-server/create_room.yaml @@ -218,6 +218,8 @@ paths: invalid: for example, the user's ``power_level`` is set below that necessary to set the room name (``errcode`` set to ``M_INVALID_ROOM_STATE``). + schema: + "$ref": "definitions/errors/error.yaml" tags: - Room creation diff --git a/api/client-server/directory.yaml b/api/client-server/directory.yaml index a50bab62..ee42cf84 100644 --- a/api/client-server/directory.yaml +++ b/api/client-server/directory.yaml @@ -68,6 +68,8 @@ paths: "errcode": "M_UNKNOWN", "error": "Room alias #monkeys:matrix.org already exists." } + schema: + "$ref": "definitions/errors/error.yaml" tags: - Room directory get: @@ -118,6 +120,8 @@ paths: "errcode": "M_NOT_FOUND", "error": "Room alias #monkeys:matrix.org not found." } + schema: + "$ref": "definitions/errors/error.yaml" tags: - Room directory delete: diff --git a/api/client-server/inviting.yaml b/api/client-server/inviting.yaml index 1e6694dc..f312d5ce 100644 --- a/api/client-server/inviting.yaml +++ b/api/client-server/inviting.yaml @@ -93,6 +93,8 @@ paths: examples: application/json: { "errcode": "M_FORBIDDEN", "error": "@cheeky_monkey:matrix.org is banned from the room"} + schema: + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: diff --git a/api/client-server/joining.yaml b/api/client-server/joining.yaml index 9890e73c..2a2a4c68 100644 --- a/api/client-server/joining.yaml +++ b/api/client-server/joining.yaml @@ -110,6 +110,8 @@ paths: examples: application/json: { "errcode": "M_FORBIDDEN", "error": "You are not invited to this room."} + schema: + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: @@ -206,6 +208,8 @@ paths: examples: application/json: { "errcode": "M_FORBIDDEN", "error": "You are not invited to this room."} + schema: + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: diff --git a/api/client-server/kicking.yaml b/api/client-server/kicking.yaml index ef43c0ee..9be5dc79 100644 --- a/api/client-server/kicking.yaml +++ b/api/client-server/kicking.yaml @@ -81,5 +81,7 @@ paths: "errcode": "M_FORBIDDEN", "error": "You do not have a high enough power level to kick from this room." } + schema: + "$ref": "definitions/errors/error.yaml" tags: - Room membership diff --git a/api/client-server/list_public_rooms.yaml b/api/client-server/list_public_rooms.yaml index 334d528c..45034488 100644 --- a/api/client-server/list_public_rooms.yaml +++ b/api/client-server/list_public_rooms.yaml @@ -59,6 +59,8 @@ paths: "errcode": "M_NOT_FOUND", "error": "Room not found" } + schema: + "$ref": "definitions/errors/error.yaml" put: summary: Sets the visibility of a room in the room directory description: |- @@ -107,6 +109,8 @@ paths: "errcode": "M_NOT_FOUND", "error": "Room not found" } + schema: + "$ref": "definitions/errors/error.yaml" "/publicRooms": get: summary: Lists the public rooms on the server. diff --git a/api/client-server/login.yaml b/api/client-server/login.yaml index e64ff953..289b17e7 100644 --- a/api/client-server/login.yaml +++ b/api/client-server/login.yaml @@ -167,12 +167,16 @@ paths: "errcode": "M_UNKNOWN", "error": "Bad login type." } + schema: + "$ref": "definitions/errors/error.yaml" 403: description: |- The login attempt failed. For example, the password may have been incorrect. examples: application/json: { "errcode": "M_FORBIDDEN"} + schema: + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: diff --git a/api/client-server/pusher.yaml b/api/client-server/pusher.yaml index 826b69d1..938014c6 100644 --- a/api/client-server/pusher.yaml +++ b/api/client-server/pusher.yaml @@ -228,7 +228,7 @@ paths: "errcode": "M_MISSING_PARAM" } schema: - type: object + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: diff --git a/api/client-server/pushrules.yaml b/api/client-server/pushrules.yaml index 22738c3e..ceb9954b 100644 --- a/api/client-server/pushrules.yaml +++ b/api/client-server/pushrules.yaml @@ -438,7 +438,7 @@ paths: "errcode": "M_UNKNOWN" } schema: - type: object + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: diff --git a/api/client-server/registration.yaml b/api/client-server/registration.yaml index 5c205999..6ae4ddd3 100644 --- a/api/client-server/registration.yaml +++ b/api/client-server/registration.yaml @@ -177,6 +177,8 @@ paths: "errcode": "M_USER_IN_USE", "error": "Desired user ID is already taken." } + schema: + "$ref": "definitions/errors/error.yaml" 401: description: |- The homeserver requires additional authentication information. @@ -249,7 +251,7 @@ paths: "error": "The specified address is already in use" } schema: - type: object + "$ref": "definitions/errors/error.yaml" "/account/password": post: summary: "Changes a user's password." @@ -420,6 +422,8 @@ paths: "errcode": "M_USER_IN_USE", "error": "Desired user ID is already taken." } + schema: + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: diff --git a/api/client-server/third_party_membership.yaml b/api/client-server/third_party_membership.yaml index 3050ee3a..66c14c4d 100644 --- a/api/client-server/third_party_membership.yaml +++ b/api/client-server/third_party_membership.yaml @@ -126,6 +126,8 @@ paths: examples: application/json: { "errcode": "M_FORBIDDEN", "error": "@cheeky_monkey:matrix.org is banned from the room"} + schema: + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: diff --git a/api/client-server/whoami.yaml b/api/client-server/whoami.yaml index 4d178e07..ad40eb86 100644 --- a/api/client-server/whoami.yaml +++ b/api/client-server/whoami.yaml @@ -65,7 +65,7 @@ paths: "error": "Unrecognised access token." } schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/error.yaml" 403: description: The appservice cannot masquerade as the user or has not registered them. @@ -75,7 +75,7 @@ paths: "error": "Application service has not registered this user." } schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: From 705423086af6ae273f8ce140f572b6e1cffa860b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 13:55:36 -0600 Subject: [PATCH 123/308] Add to changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 44d41f4e..c06c46b3 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -41,6 +41,8 @@ Unreleased changes (`#1245 `_). - Describe ``StateEvent`` for ``/createRoom`` (`#1329 `_). + - Describe the rate limit error response schema + (`#1373 `_). - Changes to the API which will be backwards-compatible for clients: From 981d1004bc9d7a82e09ca413eee87a64d80d59f1 Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Thu, 3 May 2018 23:18:47 +0900 Subject: [PATCH 124/308] Add a missing $ref --- api/client-server/message_pagination.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/api/client-server/message_pagination.yaml b/api/client-server/message_pagination.yaml index 71406653..941e61fb 100644 --- a/api/client-server/message_pagination.yaml +++ b/api/client-server/message_pagination.yaml @@ -107,6 +107,7 @@ paths: items: type: object title: RoomEvent + "$ref": "definitions/event-schemas/schema/core-event-schema/room_event.yaml" examples: application/json: { "start": "t47429-4392820_219380_26003_2265", From 0779d81e52556c28bed61b82c602fd154a8f184d Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 14:30:39 -0600 Subject: [PATCH 125/308] Clarify which requests should have CORS headers Spoilers: all of them. --- specification/client_server_api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index f7a7d509..33ee8b1a 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -169,7 +169,8 @@ Web Browser Clients It is realistic to expect that some clients will be written to be run within a web browser or similar environment. In these cases, the homeserver should respond -to pre-flight requests and supply Cross-Origin Resource Sharing (CORS) headers. +to pre-flight requests and supply Cross-Origin Resource Sharing (CORS) headers on +all requests. When a client approaches the server with a pre-flight (``OPTIONS``) request, the server should respond with the CORS headers for that route. If the route does not From 05c85fd840b3feda4916596365387e1e75cb1e8b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 14:37:21 -0600 Subject: [PATCH 126/308] Spec the 403 on GET /presence/{userId}/status --- api/client-server/presence.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/api/client-server/presence.yaml b/api/client-server/presence.yaml index ab32f983..46134430 100644 --- a/api/client-server/presence.yaml +++ b/api/client-server/presence.yaml @@ -123,6 +123,15 @@ paths: description: |- There is no presence state for this user. This user may not exist or isn't exposing presence information to you. + 403: + description: You are not allowed to see this user's presence status. + examples: + application/json: { + "errcode": "M_FORBIDDEN", + "error": "You are not allowed to see their presence" + } + schema: + "$ref": "definitions/error.yaml" tags: - Presence "/presence/list/{userId}": From 9e796192e45aad16019f29d892cf875db7eeb7c5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 15:31:23 -0600 Subject: [PATCH 127/308] Document that clients must leave before forgetting rooms Fixes https://github.com/matrix-org/matrix-doc/issues/1011 --- api/client-server/leaving.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/leaving.yaml b/api/client-server/leaving.yaml index 36351fd4..e8e58397 100644 --- a/api/client-server/leaving.yaml +++ b/api/client-server/leaving.yaml @@ -78,8 +78,8 @@ paths: for this room. If all users on a homeserver forget a room, the room is eligible for deletion from that homeserver. - If the user is currently joined to the room, they will implicitly leave - the room as part of this API call. + If the user is currently joined to the room, they must leave the room + before calling this API. operationId: forgetRoom security: - accessToken: [] From a19bbc80897fe0b4e184263200203b4b2e2c707c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 15:32:53 -0600 Subject: [PATCH 128/308] Add to changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 44d41f4e..697b02c1 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -41,6 +41,8 @@ Unreleased changes (`#1245 `_). - Describe ``StateEvent`` for ``/createRoom`` (`#1329 `_). + - Clarify that clients must leave rooms before forgetting them + (`#1378 `_). - Changes to the API which will be backwards-compatible for clients: From 9b1f3a1f4768fe153fa733e8db6d10b0c3c917e0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 18:16:24 -0600 Subject: [PATCH 129/308] Document guest_access in /createRoom presets; Make presets a table Previously the presets ended up being rendered as one long string (the indentation was not respected). This was hard to read and wasn't very clear. Now that there's a larger amount of information, it makes some sense to put it into a table. Tables don't work nicely when nested in another table, so the preset definitions have been moved into the general route description. This commit also updates the documentation to reference `preset`, not `presets` given the param is singular. Fixes https://github.com/matrix-org/matrix-doc/issues/881 --- api/client-server/create_room.yaml | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/api/client-server/create_room.yaml b/api/client-server/create_room.yaml index 3e88ee0c..37c8ec21 100644 --- a/api/client-server/create_room.yaml +++ b/api/client-server/create_room.yaml @@ -1,4 +1,5 @@ # Copyright 2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,7 +41,7 @@ paths: 0. A default ``m.room.power_levels`` event, giving the room creator (and not other members) permission to send state events. - 1. Events set by ``presets``. + 1. Events set by the ``preset``. 2. Events listed in ``initial_state``, in the order that they are listed. @@ -49,6 +50,16 @@ paths: 4. Invite events implied by ``invite`` and ``invite_3pid``. + The available presets do the following with respect to room state: + + ======================== ============== ====================== ================ ========= + Preset ``join_rules`` ``history_visibility`` ``guest_access`` Other + ======================== ============== ====================== ================ ========= + ``private_chat`` ``invite`` ``shared`` ``can_join`` + ``trusted_private_chat`` ``invite`` ``shared`` ``can_join`` All invitees are given the same power level as the room creator. + ``public_chat`` ``public`` ``shared`` ``forbidden`` + ======================== ============== ====================== ================ ========= + operationId: createRoom security: - accessToken: [] @@ -142,7 +153,7 @@ paths: room. The expected format of the state events are an object with type, state_key and content keys set. - Takes precedence over events set by ``presets``, but gets + Takes precedence over events set by ``preset``, but gets overriden by ``name`` and ``topic`` keys. items: type: object @@ -163,20 +174,7 @@ paths: enum: ["private_chat", "public_chat", "trusted_private_chat"] description: |- Convenience parameter for setting various default state events - based on a preset. Must be either: - - ``private_chat`` => - ``join_rules`` is set to ``invite``. - ``history_visibility`` is set to ``shared``. - - ``trusted_private_chat`` => - ``join_rules`` is set to ``invite``. - ``history_visibility`` is set to ``shared``. - All invitees are given the same power level as the room creator. - - ``public_chat``: => - ``join_rules`` is set to ``public``. - ``history_visibility`` is set to ``shared``. + based on a preset. is_direct: type: boolean description: |- From e409b40fad7f456f962e6005d8265de4da5cbbd0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 18:22:10 -0600 Subject: [PATCH 130/308] Add to changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 44d41f4e..781e2f90 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -17,6 +17,8 @@ Unreleased changes (`#1158 `_). - Document the GET version of ``/login`` (`#1361 `_). + - Document guest access in ``/createRoom`` presets + (`#1379 `_). - Spec clarifications: From 95e8f1bddebc927fff3bf2b03ea795a79312b2ed Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 19:21:03 -0600 Subject: [PATCH 131/308] Add to changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 44d41f4e..34e42d0b 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -39,6 +39,8 @@ Unreleased changes (`#1152 `_). - Mark ``GET /rooms/{roomId}/members`` as requiring authentication (`#1245 `_). + - Define what a ``RoomEvent`` is on ``/rooms/{roomId}/messages`` + (`#1380 `_). - Describe ``StateEvent`` for ``/createRoom`` (`#1329 `_). From 436b4106980663b595570f5878033765195be6b2 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 20:02:30 -0600 Subject: [PATCH 132/308] Document the error code when someone tries to forget a room they are in --- api/client-server/leaving.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/api/client-server/leaving.yaml b/api/client-server/leaving.yaml index e8e58397..57a148c6 100644 --- a/api/client-server/leaving.yaml +++ b/api/client-server/leaving.yaml @@ -99,6 +99,15 @@ paths: } schema: type: object + 400: + description: The user has not left the room + examples: + application/json: { + "errcode": "M_UNKNOWN", + "error": "User @example:matrix.org is in room !au1ba7o:matrix.org" + } + schema: + "$ref": "definitions/error.yaml" 429: description: This request was rate-limited. schema: From dfdc5f9bef899c356695c9cc0c5b7f9e8c45b2d5 Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Thu, 3 May 2018 23:19:55 +0900 Subject: [PATCH 133/308] search.yaml: Rename some titles to avoid confusion between request and response schemas Signed-off-by: Alexey Rusakov --- api/client-server/search.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/client-server/search.yaml b/api/client-server/search.yaml index e5f6a872..f7c96180 100644 --- a/api/client-server/search.yaml +++ b/api/client-server/search.yaml @@ -74,7 +74,7 @@ paths: properties: room_events: type: object - title: "Room Events" + title: Room Events Criteria description: Mapping of category name to search criteria. properties: search_term: @@ -103,7 +103,7 @@ paths: The order in which to search for results. By default, this is ``"rank"``. event_context: - title: "Event Context" + title: "Include Event Context" type: object description: |- Configures whether any context for the events @@ -169,13 +169,13 @@ paths: properties: search_categories: type: object - title: Categories + title: Result Categories description: Describes which categories to search in and their criteria. properties: room_events: type: object - title: Room Event Results + title: Result Room Events description: Mapping of category name to search criteria. properties: count: From 0867ab2365e403241343d45bc06c966d73442ea3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 4 Jul 2018 20:14:19 -0600 Subject: [PATCH 134/308] Add to changelog --- changelogs/client_server.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 44d41f4e..f732a6d8 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -41,6 +41,8 @@ Unreleased changes (`#1245 `_). - Describe ``StateEvent`` for ``/createRoom`` (`#1329 `_). + - Clarify the request and result types on ``/search`` + (`#1381 `_). - Changes to the API which will be backwards-compatible for clients: From 92f3989f2b00627201cb16a5fdfd2ee1fa095180 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 5 Jul 2018 11:37:06 +0100 Subject: [PATCH 135/308] Dirty replace of DNS name -> hostname --- specification/server_server_api.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 3738ea05..70d495d3 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -18,7 +18,7 @@ Federation API Matrix homeservers use the Federation APIs (also known as server-server APIs) to communicate with each other. Homeservers use these APIs to push messages to -each other in real-time, to +each other in real-time, to historic messages from each other, and to query profile and presence information about users on each other's servers. @@ -73,22 +73,22 @@ Server Discovery Resolving Server Names ~~~~~~~~~~~~~~~~~~~~~~ -Each matrix homeserver is identified by a server name consisting of a DNS name +Each matrix homeserver is identified by a server name consisting of a hostname and an optional TLS port. .. code:: - server_name = dns_name [ ":" tls_port] + server_name = hostname [ ":" tls_port] dns_name = tls_port = *DIGIT .. ** If the port is present then the server is discovered by looking up an AAAA or -A record for the DNS name and connecting to the specified TLS port. If the port +A record for the hostname and connecting to the specified TLS port. If the port is absent then the server is discovered by looking up a ``_matrix._tcp`` SRV -record for the DNS name. If this record does not exist then the server is -discovered by looking up an AAAA or A record on the DNS name and taking the +record for the hostname. If this record does not exist then the server is +discovered by looking up an AAAA or A record on the hostname and taking the default fallback port number of 8448. Homeservers may use SRV records to load balance requests between multiple TLS endpoints or to failover to another endpoint if an endpoint fails. @@ -165,7 +165,7 @@ events sent by that server can still be checked. ==================== =================== ====================================== Key Type Description ==================== =================== ====================================== -``server_name`` String DNS name of the homeserver. +``server_name`` String hostname of the homeserver. ``verify_keys`` Object Public keys of the homeserver for verifying digital signatures. ``old_verify_keys`` Object The public keys that the server used @@ -273,7 +273,7 @@ at ``/_matrix/key/v1``. ==================== =================== ====================================== Key Type Description ==================== =================== ====================================== -``server_name`` String DNS name of the homeserver. +``server_name`` String hostname of the homeserver. ``verify_keys`` Object Public keys of the homeserver for verifying digital signatures. ``signatures`` Object Digital signatures for this object @@ -897,10 +897,10 @@ the resident homeserver. The required fields are: ======================== ============ ========================================= ``type`` String The value ``m.room.member``. ``auth_events`` List An event-reference list containing the - authorization events that would allow + authorization events that would allow this member to join. ``content`` Object The event content. -``depth`` Integer (this field must be present but is +``depth`` Integer (this field must be present but is ignored; it may be 0) ``origin`` String The name of the resident homeserver. ``origin_server_ts`` Integer A timestamp added by the resident From cc3724b54a44bef1ee26577e74b2f167aebdcc2a Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 Jul 2018 11:55:25 +0100 Subject: [PATCH 136/308] fix indentation, error messages. add rest of PR --- .../application_service.yaml | 331 ++++---- api/client-server/third_party_lookup.yaml | 338 ++++++++ changelogs/client_server.rst | 2 - specification/proposals.rst | 770 +++++++++++++++++- specification/targets.yaml | 2 +- specification/third_party_lookup.rst | 26 + 6 files changed, 1306 insertions(+), 163 deletions(-) create mode 100644 api/client-server/third_party_lookup.yaml create mode 100644 specification/third_party_lookup.rst diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index a0713d45..5efc058b 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -30,8 +30,8 @@ paths: put: summary: Send some events to the application service. description: |- - This API is called by the HS when the HS wants to push an event (or - batch of events) to the AS. + This API is called by the homeserver when it wants to push an event + (or batch of events) to the application service. operationId: sendTransaction parameters: - in: path @@ -48,33 +48,33 @@ paths: schema: type: object example: { - "events": [ - { - "age": 32, - "content": { - "body": "incoming message", - "msgtype": "m.text" - }, - "event_id": "$14328055551tzaee:localhost", - "origin_server_ts": 1432804485886, - "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", - "type": "m.room.message", - "user_id": "@bob:localhost" + "events": [ + { + "age": 32, + "content": { + "body": "incoming message", + "msgtype": "m.text" }, - { - "age": 1984, - "content": { - "body": "another incoming message", - "msgtype": "m.text" - }, - "event_id": "$1228055551ffsef:localhost", - "origin_server_ts": 1432804485886, - "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", - "type": "m.room.message", - "user_id": "@bob:localhost" - } - ] - } + "event_id": "$14328055551tzaee:localhost", + "origin_server_ts": 1432804485886, + "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", + "type": "m.room.message", + "user_id": "@bob:localhost" + }, + { + "age": 1984, + "content": { + "body": "another incoming message", + "msgtype": "m.text" + }, + "event_id": "$1228055551ffsef:localhost", + "origin_server_ts": 1432804485886, + "room_id": "!TmaZBKYIFrIPVGoUYp:localhost", + "type": "m.room.message", + "user_id": "@bob:localhost" + } + ] + } description: "Transaction informations" properties: events: @@ -89,14 +89,14 @@ paths: description: The transaction was processed successfully. examples: application/json: { - } + } schema: type: object 500: description: There was a problem completing the request. examples: application/json: { - } + } schema: type: object "/rooms/{roomAlias}": @@ -126,7 +126,7 @@ paths: before responding. examples: application/json: { - } + } schema: type: object 401: @@ -135,8 +135,8 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } schema: type: object 403: @@ -144,8 +144,8 @@ paths: The credentials supplied by the homeserver were rejected. examples: application/json: { - "errcode": "M_FORBIDDEN" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" + } schema: type: object 404: @@ -154,15 +154,15 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "M_NOT_FOUND" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } schema: type: object 500: description: There was a problem completing the request. examples: application/json: { - } + } schema: type: object "/users/{userId}": @@ -189,7 +189,7 @@ paths: service MUST create the user using the client-server API. examples: application/json: { - } + } schema: type: object 401: @@ -198,8 +198,8 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } schema: type: object 403: @@ -207,8 +207,8 @@ paths: The credentials supplied by the homeserver were rejected. examples: application/json: { - "errcode": "M_FORBIDDEN" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" + } schema: type: object 404: @@ -217,24 +217,24 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "M_NOT_FOUND" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } schema: type: object 500: description: There was a problem completing the request. examples: application/json: { - } + } schema: type: object "/_matrix/app/unstable/thirdparty/{protocol}": get: summary: Retrieve metadata about a specific protocol that the application service supports. description: |- - This API is called by the HS when it wants to present clients with - specific information about the various thirdparty networks that an AS - supports. + This API is called by the homeserver when it wants to present clients + with specific information about the various third party networks that + an application service supports. operationId: queryMetadata parameters: - in: path @@ -249,33 +249,33 @@ paths: description: The protocol was found and metadata returned. examples: application/json: { - "user_fields": ["network", "nickname"], - "location_fields": ["network", "channel"], - "icon": "mxc://example.org/aBcDeFgH", - "field_types": { - "network": { - "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", - "placeholder": "irc.example.org" - }, - "nickname": { - "regexp": "[^\\s]+", - "placeholder": "username" - }, - "channel": { - "regexp": "#[^\\s]+", - "placeholder": "#foobar" - } + "user_fields": ["network", "nickname"], + "location_fields": ["network", "channel"], + "icon": "mxc://example.org/aBcDeFgH", + "field_types": { + "network": { + "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", + "placeholder": "irc.example.org" }, - "instances": [ - { - "desc": "Freenode", - "icon": "mxc://example.org/JkLmNoPq", - "fields": { - "network": "freenode.net", - } + "nickname": { + "regexp": "[^\\s]+", + "placeholder": "username" + }, + "channel": { + "regexp": "#[^\\s]+", + "placeholder": "#foobar" + } + }, + "instances": [ + { + "desc": "Freenode", + "icon": "mxc://example.org/JkLmNoPq", + "fields": { + "network": "freenode.net", } - ] - } + } + ] + } schema: type: object 401: @@ -284,8 +284,8 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } schema: type: object 403: @@ -293,30 +293,30 @@ paths: The credentials supplied by the homeserver were rejected. examples: application/json: { - "errcode": "M_FORBIDDEN" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" + } schema: type: object 404: description: No protocol was found with the given path. examples: application/json: { - "errcode": "M_NOT_FOUND" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } schema: type: object 500: description: There was a problem completing the request. examples: application/json: { - } + } schema: type: object "/_matrix/app/unstable/thirdparty/user/{protocol}": get: - summary: Retrieve the Matrix ID of a corresponding thirdparty user. + summary: Retrieve the Matrix ID of a corresponding third party user. description: |- - This API is called by the HS in order to retrieve a Matrix ID linked + This API is called by the homeserver in order to retrieve a Matrix ID linked to a user on the external service, given a set of user parameters. operationId: queryUserByProtocol parameters: @@ -331,32 +331,47 @@ paths: name: field1, field2... type: string description: |- - One or more custom fields that are passed to the AS to help identify the user. + One or more custom fields that are passed to the application + service to help identify the user. responses: 200: description: The Matrix IDs found with the given parameters. examples: application/json: [{ - "userid": "@_gitter_jim:matrix.org", - "protocol": "gitter", - "fields": { - "user": "jim" - } - }] + "userid": "@_gitter_jim:matrix.org", + "protocol": "gitter", + "fields": { + "user": "jim" + } + }] schema: type: array description: Matched users. items: type: object title: User + properties: + userid: + type: string + description: The Matrix ID of the matched user. + protocol: + type: string + description: The third party protocol. + fields: + type: object + description: The third party network fields used to identify this user. + properties: + user: + type: string + description: An example field, in this case the username for a Gitter user. 401: description: |- The homeserver has not supplied credentials to the application service. Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } schema: type: object 403: @@ -364,67 +379,67 @@ paths: The credentials supplied by the homeserver were rejected. examples: application/json: { - "errcode": "M_FORBIDDEN" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" + } schema: type: object 404: description: No users were found with the given parameters. examples: application/json: { - "errcode": "M_NOT_FOUND" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } schema: type: object 500: description: There was a problem completing the request. examples: application/json: { - } + } schema: type: object "/_matrix/app/unstable/thirdparty/location/{protocol}": get: - summary: Retreive Matrix-side portals rooms leading to a thirdparty location. + summary: Retreive Matrix-side portal rooms leading to a third party location. description: |- Requesting this endpoint with a valid protocol name results in a list of successful mapping results in a JSON array. Each result contains objects to represent the Matrix room or rooms that represent a portal - to this 3PN. Each has the Matrix room alias string, an identifier for - the particular 3PN protocol, and an object containing the - network-specific fields that comprise this identifier. It should - attempt to canonicalise the identifier as much as reasonably possible - given the network type. + to this third party network. Each has the Matrix room alias string, + an identifier for the particular third party network protocol, and an + object containing the network-specific fields that comprise this + identifier. It should attempt to canonicalise the identifier as much + as reasonably possible given the network type. operationId: queryLocationByProtocol parameters: - in: path name: protocol type: string - description: The protocol used to communicate to the thirdparty network. + description: The protocol used to communicate to the third party network. required: true x-example: "irc" - in: query name: field1, field2... type: string description: |- - One or more custom fields that are passed to the AS to help - identify the thirdparty location. + One or more custom fields that are passed to the application + service to help identify the third party location. responses: 200: description: At least one portal room was found. examples: application/json: [{ - "alias": "#freenode_#matrix:matrix.org", - "protocol": "irc", - "fields": { - "network": "freenode", - "channel": "#matrix" - } - }] + "alias": "#freenode_#matrix:matrix.org", + "protocol": "irc", + "fields": { + "network": "freenode", + "channel": "#matrix" + } + }] schema: type: array description: |- - Array of portal rooms leading to the requested thirdparty + Array of portal rooms leading to the requested third party location. items: type: object @@ -435,8 +450,8 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } schema: type: object 403: @@ -444,54 +459,54 @@ paths: The credentials supplied by the homeserver were rejected. examples: application/json: { - "errcode": "M_FORBIDDEN" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" + } schema: type: object 404: description: No mappings were found with the given parameters. examples: application/json: { - "errcode": "M_NOT_FOUND" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } schema: type: object 500: description: There was a problem completing the request. examples: application/json: { - } + } schema: type: object "/_matrix/app/unstable/thirdparty/location": get: - summary: Reverse-lookup thirdparty locations given a Matrix room alias. + summary: Reverse-lookup third party locations given a Matrix room alias. description: |- - Retreive an array of thirdparty network locations from a Matrix room + Retreive an array of third party network locations from a Matrix room alias. operationId: queryLocationByAlias parameters: - - in: path + - in: query name: alias type: string description: The Matrix room alias to look up. responses: 200: description: |- - All found thirdparty locations. Same response format as the + All found third party locations. Same response format as the forward lookup response. examples: application/json: [{ - "alias": "#freenode_#matrix:matrix.org", - "protocol": "irc", - "fields": { - "network": "freenode", - "channel": "#matrix" - } - }] + "alias": "#freenode_#matrix:matrix.org", + "protocol": "irc", + "fields": { + "network": "freenode", + "channel": "#matrix" + } + }] schema: type: array - description: Matched thirdparty locations. + description: Matched third party locations. items: type: object title: Location @@ -501,8 +516,8 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } schema: type: object 403: @@ -510,51 +525,51 @@ paths: The credentials supplied by the homeserver were rejected. examples: application/json: { - "errcode": "M_FORBIDDEN" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" + } schema: type: object 404: description: No mappings were found with the given parameters. examples: application/json: { - "errcode": "M_NOT_FOUND" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } schema: type: object 500: description: There was a problem completing the request. examples: application/json: { - } + } schema: type: object "/_matrix/app/unstable/thirdparty/user": get: - summary: Reverse-lookup thirdparty users given a Matrix ID. + summary: Reverse-lookup third party users given a Matrix ID. description: |- - Retreive an array of thirdparty users from a Matrix ID. + Retreive an array of third party users from a Matrix ID. operationId: queryUserByID paramters: - - in: path + - in: query name: userid type: string description: The Matrix ID to look up. responses: 200: description: |- - An array of thirdparty users. + An array of third party users. examples: application/json: [{ - "userid": "@_gitter_jim:matrix.org", - "protocol": "gitter", - "fields": { - "user": "jim" - } - }] + "userid": "@_gitter_jim:matrix.org", + "protocol": "gitter", + "fields": { + "user": "jim" + } + }] schema: type: array - description: Matched thirdparty users + description: Matched third party users items: type: object title: User @@ -564,8 +579,8 @@ paths: Optional error information can be included in the body of this response. examples: application/json: { - "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } schema: type: object 403: @@ -573,22 +588,22 @@ paths: The credentials supplied by the homeserver were rejected. examples: application/json: { - "errcode": "M_FORBIDDEN" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" + } schema: type: object 404: description: No mappings were found with the given parameters. examples: application/json: { - "errcode": "M_NOT_FOUND" - } + "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" + } schema: type: object 500: description: There was a problem completing the request. examples: application/json: { - } + } schema: type: object \ No newline at end of file diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml new file mode 100644 index 00000000..41b5df16 --- /dev/null +++ b/api/client-server/third_party_lookup.yaml @@ -0,0 +1,338 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +swagger: '2.0' +info: + title: "Matrix Client-Server Third Party Lookup API" + version: "1.0.0" +host: localhost:8008 +schemes: + - https + - http +basePath: /_matrix/client/%CLIENT_MAJOR_VERSION% +consumes: + - application/json +produces: + - application/json +paths: + "/thirdparty/protocols": + get: + summary: Retrieve metadata about all protocols that a homeserver supports. + description: |- + Fetches the overall metadata about protocols supported by the + homeserver. Includes both the available protocols and all fields + required for queries against each protocol. + operationId: queryMetadata + responses: + 200: + description: The protocols supported by the homeserver. + examples: + application/json: { + "irc": { + "user_fields": ["network", "nickname"], + "location_fields": ["network", "channel"], + "icon": "mxc://example.org/aBcDeFgH", + "field_types": { + "network": { + "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", + "placeholder": "irc.example.org" + }, + "nickname": { + "regexp": "[^\\s]+", + "placeholder": "username" + }, + "channel": { + "regexp": "#[^\\s]+", + "placeholder": "#foobar" + } + }, + "instances": [ + { + "desc": "Freenode", + "icon": "mxc://example.org/JkLmNoPq", + "fields": { + "network": "freenode.net", + } + } + ] + }, + "gitter": { + "user_fields": ["username"], + "location_fields": ["room"], + "field_types": { + "username": { + "regexp": "@[^\\s]+", + "placeholder": "@username" + }, + "room": { + "regexp": "[^\\s]+\\/[^\\s]+", + "placeholder": "matrix-org/matrix-doc" + } + }, + "instances": [ + { + "desc": "Gitter", + "icon": "mxc://example.org/zXyWvUt", + "fields": {} + } + ] + } + } + schema: + type: object + "/thirdparty/protocol/{protocol}": + get: + summary: Retrieve metadata about a specific protocol that the homeserver supports. + description: |- + Fetches the metadata from the homeserver about a particular third party protocol. + operationId: queryMetadata + parameters: + - in: path + name: protocol + type: string + description: |- + The name of the protocol. + required: true + x-example: "irc" + responses: + 200: + description: The protocol was found and metadata returned. + examples: + application/json: { + "user_fields": ["network", "nickname"], + "location_fields": ["network", "channel"], + "icon": "mxc://example.org/aBcDeFgH", + "field_types": { + "network": { + "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", + "placeholder": "irc.example.org" + }, + "nickname": { + "regexp": "[^\\s#]+", + "placeholder": "username" + }, + "channel": { + "regexp": "#[^\\s]+", + "placeholder": "#foobar" + } + }, + "instances": [ + { + "desc": "Freenode", + "icon": "mxc://example.org/JkLmNoPq", + "fields": { + "network": "freenode.net", + } + } + ] + } + schema: + type: object + 404: + description: The protocol is unknown + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object + "/thirdparty/location/{protocol}": + get: + summary: Retreive Matrix-side portals rooms leading to a third party location. + description: |- + Requesting this endpoint with a valid protocol name results in a list + of successful mapping results in a JSON array. Each result contains + objects to represent the Matrix room or rooms that represent a portal + to this third party network. Each has the Matrix room alias string, + an identifier for the particular third party network protocol, and an + object containing the network-specific fields that comprise this + identifier. It should attempt to canonicalise the identifier as much + as reasonably possible given the network type. + operationId: queryLocationByProtocol + parameters: + - in: path + name: protocol + type: string + description: The protocol used to communicate to the third party network. + required: true + x-example: "irc" + - in: query + name: field1, field2... + type: string + description: |- + One or more custom fields to help identify the third party + location. + responses: + 200: + description: At least one portal room was found. + examples: + application/json: [{ + "alias": "#freenode_#matrix:matrix.org", + "protocol": "irc", + "fields": { + "network": "freenode", + "channel": "#matrix" + } + }] + schema: + type: array + description: |- + Array of portal rooms leading to the requested third party + location. + items: + type: object + title: Portal Room + 404: + description: The Matrix room alias was not found + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object + "/thirdparty/user/{protocol}": + get: + summary: Retrieve the Matrix ID of a corresponding third party user. + description: |- + Retrieve a Matrix ID linked to a user on the third party service, given + a set of user parameters. + operationId: queryUserByProtocol + parameters: + - in: path + name: protocol + type: string + description: |- + The name of the protocol. + required: true + x-example: irc + - in: query + name: field1, field2... + type: string + description: |- + One or more custom fields that are passed to the AS to help identify the user. + responses: + 200: + description: The Matrix IDs found with the given parameters. + examples: + application/json: [{ + "userid": "@_gitter_jim:matrix.org", + "protocol": "gitter", + "fields": { + "user": "jim" + } + }] + schema: + type: array + description: Matched users. + items: + type: object + title: User + properties: + userid: + type: string + description: The Matrix ID of the matched user. + protocol: + type: string + description: The third party protocol. + fields: + type: object + description: The third party network fields used to identify this user. + properties: + user: + type: string + description: An example field, in this case the username for a Gitter user. + 404: + description: The Matrix ID was not found + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object + "/thirdparty/location": + get: + summary: Reverse-lookup third party locations given a Matrix room alias. + description: |- + Retreive an array of third party network locations from a Matrix room + alias. + operationId: queryLocationByAlias + parameters: + - in: path + name: alias + type: string + description: The Matrix room alias to look up. + responses: + 200: + description: |- + All found third party locations. Same response format as the + forward lookup response. + examples: + application/json: [{ + "alias": "#freenode_#matrix:matrix.org", + "protocol": "irc", + "fields": { + "network": "freenode", + "channel": "#matrix" + } + }] + schema: + type: array + description: Matched third party locations. + items: + type: object + title: Location + 404: + description: The Matrix room alias was not found + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object + "/thirdparty/user": + get: + summary: Reverse-lookup third party users given a Matrix ID. + description: |- + Retreive an array of third party users from a Matrix ID. + operationId: queryUserByID + paramters: + - in: path + name: userid + type: string + description: The Matrix ID to look up. + responses: + 200: + description: |- + An array of third party users. + examples: + application/json: [{ + "userid": "@_gitter_jim:matrix.org", + "protocol": "gitter", + "fields": { + "user": "jim" + } + }] + schema: + type: array + description: Matched third party users + items: + type: object + title: User + 404: + description: The Matrix ID was not found + examples: + application/json: { + "errcode": "M_NOT_FOUND" + } + schema: + type: object \ No newline at end of file diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index f7e94dbe..7f31aa75 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -11,8 +11,6 @@ Unreleased changes (`#1110 `_). - ``POST /delete_devices`` (`#1239 `_). - - ``GET /thirdparty/protocols`` - (`#1353 `_). - ``GET /thirdparty/protocol/{protocol}`` (`#1353 `_). - ``GET /thirdparty/location/{protocol}`` diff --git a/specification/proposals.rst b/specification/proposals.rst index 371850ab..d04edfa8 100644 --- a/specification/proposals.rst +++ b/specification/proposals.rst @@ -1,6 +1,772 @@ Tables of Tracked Proposals --------------------------- -This file is autogenerated by a jenkins build process +proposal-wip +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -View the current live version `at https://matrix.org/docs/spec/proposals `_ +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documentation + - Author + - Shepherd + - PRs + * - `MSC455 `_ + - Do we want to specify a matrix:// URI scheme for rooms? (SPEC-5) + - 2014-09-15 + - 2018-05-15 + - `455-1 `_ + - `@KitsuneRal`_ + - None + - + * - `MSC586 `_ + - Federation API for canonicalising MXIDs + - 2014-10-27 + - 2018-05-15 + - `586-1 `_ + - `@ara4n`_ + - None + - + * - `MSC489 `_ + - Extensible Profiles. (SPEC-93) + - 2015-01-19 + - 2018-05-15 + - `489-1 `_ + - `@erikjohnston`_ + - None + - + * - `MSC1196 `_ + - Matrix Escape Hatch (Versioned Rooms) + - 2015-10-22 + - 2018-05-15 + - `1196-1 `_ + - `@leonerd`_ + - None + - + * - `MSC1148 `_ + - Support for websockets + - 2015-11-16 + - 2018-06-04 + - `1148-1 `_, `1148-2 `_ + - `@richvdh`_, `@krombel`_ + - None + - + * - `MSC1238 `_ + - Push to Talk + - 2016-04-13 + - 2018-05-15 + - `1238-1 `_ + - `@aviraldg`_ + - None + - `PR#310`_ + * - `MSC1198 `_ + - Threading API + - 2016-05-23 + - 2018-05-15 + - `1198-1 `_ + - `@Half-Shot`_ + - None + - + * - `MSC1207 `_ + - Publishing Room Lists for 3rd party networks + - 2016-10-21 + - 2018-05-31 + - `1207-1 `_ + - `@erikjohnston`_ + - None + - + * - `MSC441 `_ + - Support for Reactions / Aggregations + - 2016-12-25 + - 2018-05-15 + - `441-1 `_ + - `@pik`_ + - `@ara4n`_ + - + * - `MSC1237 `_ + - Improving m.location with GeoJSON and accuracy + - 2017-05-19 + - 2018-05-15 + - `1237-1 `_ + - `@Half-Shot`_ + - None + - `PR#919`_ + * - `MSC971 `_ + - Add groups stuff to spec + - 2017-08-10 + - 2018-05-20 + - `971-1 `_, `971-2 `_, `971-3 `_ + - `@erikjohnston`_ + - None + - + * - `MSC1215 `_ + - Groups as Rooms + - 2017-10-17 + - 2018-05-15 + - `1215-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1217 `_ + - Visibility of groups to group members and their non-members + - 2017-10-30 + - 2018-05-15 + - `1217-1 `_ + - `@lampholder`_ + - None + - + * - `MSC1218 `_ + - Bridging group membership with 3rd party group sources + - 2017-11-15 + - 2018-05-15 + - `1218-1 `_ + - `@lukebarnard1`_ + - None + - + * - `MSC1219 `_ + - Proposal for storing megolm keys serverside + - 2017-11-23 + - 2018-05-15 + - `1219-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1221 `_ + - Improving Presence + - 2017-12-26 + - 2018-05-24 + - `1221-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1222 `_ + - Pushing updates about Groups (Communities) to clients + - 2018-01-02 + - 2018-05-24 + - `1222-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1225 `_ + - Extensible event types & fallback in Matrix + - 2018-02-18 + - 2018-05-15 + - `1225-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1206 `_ + - Proposal for improved bot support + - 2018-03-14 + - 2018-06-08 + - `1206-1 `_ + - `@turt2live`_ + - None + - + * - `MSC1228 `_ + - Removing MXIDs from events + - 2018-04-19 + - 2018-05-15 + - `1228-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1231 `_ + - Handling Consent for Privacy Policy + - 2018-05-02 + - 2018-05-15 + - `1231-1 `_ + - `@neilisfragile`_ + - None + - + * - `MSC1267 `_ + - Interactive Key Verification + - 2018-05-28 + - 2018-05-28 + - `1267-1 `_ + - `@uhoreg`_ + - None + - + * - `MSC1280 `_ + - Mechanisms for communicating erasure requests to bots and federated homeservers + - 2018-06-05 + - 2018-06-05 + - `1280-1 `_ + - `@richvdh`_ + - None + - + * - `MSC688 `_ + - Calculate room names server-side + - 2018-06-10 + - 2018-06-15 + - `688-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1324 `_ + - Custom protocol definitions for application services + - 2018-06-20 + - 2018-06-20 + - `1324-1 `_ + - `@anoadragon453`_ + - None + - + * - `MSC1323 `_ + - AS traffic rate-limiting + - 2018-06-20 + - 2018-07-03 + - `1323-1 `_ + - `@anoadragon453`_ + - None + - + * - `MSC1322 `_ + - Mechanism to communicate 3PID binding updates or deletions to homeservers + - 2018-06-20 + - 2018-06-20 + - `1322-1 `_ + - `@babolivier`_ + - None + - + + + +proposal-ready-for-review +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documentation + - Author + - Shepherd + - PRs + * - `MSC1227 `_ + - Proposal for lazy-loading room members to improve initial sync speed and client RAM usage + - 2018-03-05 + - 2018-06-10 + - `1227-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1206 `_ + - Proposal for improved bot support + - 2018-03-14 + - 2018-06-08 + - `1206-1 `_ + - `@turt2live`_ + - None + - + * - `MSC1256 `_ + - Custom emoji and sticker packs in matrix + - 2018-05-23 + - 2018-05-24 + - `1256-1 `_ + - `@turt2live`_ + - None + - + * - `MSC1270 `_ + - Proposal Add /_matrix/media/v1/resolve_url to Client-Server API: download and preview urls in the clients despite CORS + - 2018-05-31 + - 2018-06-19 + - `1270-1 `_ + - `@oivoodoo`_ + - None + - + * - `MSC701 `_ + - Auth for content repo (and enforcing GDPR erasure) + - 2018-06-04 + - 2018-06-07 + - `701-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1304 `_ + - Proposal to simplify the auth rules of m.room.power_level events. + - 2018-06-13 + - 2018-06-14 + - `1304-1 `_ + - `@richvdh`_, `@ara4n`_ + - None + - + * - `MSC1301 `_ + - Proposal for improving authorization for the matrix profile API + - 2018-06-13 + - 2018-06-13 + - `1301-1 `_ + - `@turt2live`_ + - None + - + * - `MSC1309 `_ + - Proposal for an application service management API + - 2018-06-14 + - 2018-06-15 + - `1309-1 `_ + - `@turt2live`_ + - None + - + * - `MSC1308 `_ + - Proposal for speeding up review of simple spec changes + - 2018-06-14 + - 2018-06-24 + - + - `@ara4n`_ + - None + - + * - `MSC1306 `_ + - Proposal to filter out traffic to Appservices based on filters + - 2018-06-14 + - 2018-06-14 + - `1306-1 `_ + - `@Half-Shot`_ + - None + - + + + +proposal-in-review +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documentation + - Author + - Shepherd + - PRs + * - `MSC1194 `_ + - A way for HSes to remove bindings from ISes (aka unbind) + - 2018-05-09 + - 2018-06-05 + - `1194-1 `_ + - `@dbkr`_ + - None + - + + + +proposal-passed-review +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documentation + - Author + - Shepherd + - PRs + * - `MSC433 `_ + - Support for discovering API endpoints via .well-known URIs (SPEC-121) + - 2015-03-08 + - 2018-07-01 + - `433-1 `_, `433-2 `_ + - `@maxidor`_, `others`_ + - `@uhoreg`_ + - + * - `MSC1226 `_ + - State Reset mitigation proposal + - 2018-02-20 + - 2018-05-15 + - `1226-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1229 `_ + - Mitigating abuse of the event depth parameter over federation + - 2018-04-30 + - 2018-05-15 + - `1229-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1232 `_ + - Media limits API + - 2018-05-04 + - 2018-06-21 + - `1232-1 `_ + - `@Half-Shot`_ + - None + - `PR#1189`_ + * - `MSC1236 `_ + - Matrix Widget API v2 + - 2018-05-13 + - 2018-05-15 + - `1236-1 `_ + - `@rxl881`_ + - None + - + + + +spec-pr-missing +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documentation + - Author + - Shepherd + - PRs + * - `MSC1200 `_ + - Configuration of E2E encryption in a room + - 2016-06-16 + - 2018-05-31 + - `1200-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1201 `_ + - Device Management API + - 2016-07-14 + - 2018-05-15 + - `1201-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1203 `_ + - 3rd Party Entity lookup API + - 2016-07-21 + - 2018-07-02 + - `1203-1 `_ + - `@leonerd`_ + - None + - + * - `MSC1208 `_ + - Encrypted attachment format + - 2016-10-26 + - 2018-05-15 + - `1208-1 `_ + - `@NegativeMjark`_ + - None + - + * - `MSC739 `_ + - Reporting inappropriate content in Matrix + - 2016-11-21 + - 2018-05-31 + - `739-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1211 `_ + - Megolm session export format + - 2017-01-03 + - 2018-05-15 + - `1211-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1212 `_ + - Device List Update Stream + - 2017-01-18 + - 2018-05-15 + - `1212-1 `_ + - `@richvdh`_ + - None + - + * - `MSC829 `_ + - Need to spec msisdn login API + - 2017-03-08 + - 2018-05-15 + - `829-1 `_ + - `@dbkr`_ + - None + - + * - `MSC855 `_ + - spec m.login.msisdn UI auth type + - 2017-03-24 + - 2018-05-15 + - `855-1 `_ + - `@dbkr`_ + - None + - + * - `MSC910 `_ + - Add new Read Marker API to docs + - 2017-05-08 + - 2018-05-15 + - + - `@lukebarnard1`_ + - None + - + * - `MSC1067 `_ + - Spec @mentions + - 2017-07-14 + - 2018-05-15 + - `1067-1 `_ + - `@lukebarnard1`_ + - None + - + * - `MSC1214 `_ + - Related Groups (i.e. flair) + - 2017-10-03 + - 2018-05-15 + - `1214-1 `_ + - `@lukebarnard1`_ + - None + - + * - `MSC1033 `_ + - Doc @room notifications + - 2017-10-23 + - 2018-05-31 + - + - `@dbkr`_ + - None + - + * - `MSC1183 `_ + - Document key-share requests + - 2018-04-30 + - 2018-05-31 + - `1183-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1230 `_ + - Temporary mitigation for depth parameter abuse + - 2018-05-01 + - 2018-05-15 + - `1230-1 `_ + - `@ara4n`_ + - None + - + * - `MSC1234 `_ + - Rich Replies format + - 2018-05-12 + - 2018-05-18 + - `1234-1 `_ + - `@t3chguy`_ + - None + - + + + +merged +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documentation + - Author + - Shepherd + - PRs + * - `MSC1197 `_ + - Ignoring Users + - 2016-05-03 + - 2018-05-18 + - `1197-1 `_ + - `@erikjohnston`_ + - None + - `PR#1142`_ + * - `MSC1199 `_ + - Notifications API + - 2016-05-23 + - 2018-06-25 + - `1199-1 `_ + - `@dbkr`_ + - None + - + * - `MSC1204 `_ + - Access Token Semantics (refresh and macaroons) - aka Auth Sept 2016 Edition + - 2016-09-29 + - 2018-06-25 + - `1204-1 `_ + - `@richvdh`_ + - None + - + * - `MSC1205 `_ + - Proposal for multi-device deletion API + - 2016-10-12 + - 2018-06-25 + - `1205-1 `_ + - `@richvdh`_ + - None + - `PR#1239`_ + * - `MSC953 `_ + - Add /user_directory/search API + - 2017-05-31 + - 2018-05-10 + - `953-1 `_ + - `@erikjohnston`_ + - None + - + * - `MSC1233 `_ + - A proposal for organising spec proposals + - 2018-05-10 + - 2018-06-25 + - `1233-1 `_ + - `@ara4n`_ + - None + - `PR#1240`_ + + + +abandoned +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documentation + - Author + - Shepherd + - PRs + * - `MSC531 `_ + - Homeservers as OAuth authorization endpoints (resource owners) (SPEC-206) + - 2015-07-25 + - 2018-05-15 + - `531-1 `_ + - `@Kegsay`_ + - None + - + * - `MSC1202 `_ + - Profile Personae + - 2016-07-15 + - 2018-05-15 + - `1202-1 `_ + - `@erikjohnston`_ + - None + - + * - `MSC1209 `_ + - Server Side Profile API + - 2016-11-01 + - 2018-05-15 + - `1209-1 `_ + - `@erikjohnston`_ + - None + - + * - `MSC1213 `_ + - Set defaults for m.federate + - 2017-04-10 + - 2018-05-18 + - `1213-1 `_ + - `@psaavedra`_ + - None + - + + + +obsolete +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: auto + :stub-columns: 1 + + * - MSC + - Proposal Title + - Creation Date + - Update Date + - Documentation + - Author + - Shepherd + - PRs + * - `MSC1223 `_ + - Replies event format + - 2018-02-01 + - 2018-05-15 + - `1223-1 `_ + - `@t3chguy`_ + - None + - + * - `MSC1224 `_ + - Replies - next steps + - 2018-02-03 + - 2018-05-15 + - `1224-1 `_ + - `@t3chguy`_ + - None + - + * - `MSC1235 `_ + - Proposal for Calendar Events + - 2018-02-06 + - 2018-05-15 + - `1235-1 `_ + - `@Half-Shot`_ + - None + - + * - `MSC1220 `_ + - Rich quoting proposal + - 2018-05-10 + - 2018-05-15 + - `1220-1 `_ + - `@t3chguy`_ + - None + - + + + + + +.. _@rxl881: https://github.com/rxl881 +.. _@turt2live: https://github.com/turt2live +.. _@erikjohnston: https://github.com/erikjohnston +.. _@anoadragon453: https://github.com/anoadragon453 +.. _@t3chguy: https://github.com/t3chguy +.. _@Kegsay: https://github.com/Kegsay +.. _@KitsuneRal: https://github.com/KitsuneRal +.. _@leonerd: https://github.com/leonerd +.. _@psaavedra: https://github.com/psaavedra +.. _@ara4n: https://github.com/ara4n +.. _@krombel: https://github.com/krombel +.. _@maxidor: https://github.com/maxidor +.. _@uhoreg: https://github.com/uhoreg +.. _@pik: https://github.com/pik +.. _@neilisfragile: https://github.com/neilisfragile +.. _@babolivier: https://github.com/babolivier +.. _@lukebarnard1: https://github.com/lukebarnard1 +.. _others: https://github.com/thers +.. _@Half-Shot: https://github.com/Half-Shot +.. _@aviraldg: https://github.com/aviraldg +.. _@oivoodoo: https://github.com/oivoodoo +.. _@richvdh: https://github.com/richvdh +.. _@NegativeMjark: https://github.com/NegativeMjark +.. _@lampholder: https://github.com/lampholder +.. _@dbkr: https://github.com/dbkr +.. _PR#1189: https://github.com/matrix-org/matrix-doc/pull/1189 +.. _PR#310: https://github.com/matrix-org/matrix-doc/pull/310 +.. _PR#1142: https://github.com/matrix-org/matrix-doc/pull/1142 +.. _PR#1239: https://github.com/matrix-org/matrix-doc/pull/1239 +.. _PR#1240: https://github.com/matrix-org/matrix-doc/pull/1240 +.. _PR#919: https://github.com/matrix-org/matrix-doc/pull/919 \ No newline at end of file diff --git a/specification/targets.yaml b/specification/targets.yaml index c6c9cd23..96199d29 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -12,7 +12,7 @@ targets: - { 1: modules.rst } - { 2: feature_profiles.rst } - { 2: "group:modules" } # reference a group of files - - { 1: thirdparty_lookup.rst } + - { 1: third_party_lookup.rst } version_label: "%CLIENT_RELEASE_LABEL%" application_service: files: diff --git a/specification/third_party_lookup.rst b/specification/third_party_lookup.rst new file mode 100644 index 00000000..1279f9fe --- /dev/null +++ b/specification/third_party_lookup.rst @@ -0,0 +1,26 @@ +Application Services +==================== + +An application service is a separate program that interacts with a homeserver +and provides various bits of functionality that would otherwise not make +sense to include directly in the homeserver. This ranges from bots, which can +often be interacted with, to bridges, which allow Matrix users to communicate +with users on third party networks. The following describes endpoints that a +Matrix client can use to interact with an application service through the +facilitation of the homeserver. + +Third Party Lookups +------------------ + +Application services can provide access to third party networks via bridging. +This allows Matrix users to communicate with users on other communication +platforms, with messages ferried back and forth by the application service. A +single application service may bridge multiple third party networks, and many +individual locations within those networks. A single third party network +location may be bridged to multiple Matrix rooms. + +In order for a client to join one of these bridged rooms, or communicate +directly with a user on a third party network, the following endpoints can be +used. + +{{third_party_lookup_cs_http_api}} \ No newline at end of file From a6dad76c018925867b4d714a2a87e502aff623b9 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 Jul 2018 11:56:30 +0100 Subject: [PATCH 137/308] Fix indentation --- .../application_service.yaml | 28 +++++++++---------- api/client-server/third_party_lookup.yaml | 28 +++++++++---------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index 5efc058b..5edb1012 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -350,20 +350,20 @@ paths: items: type: object title: User - properties: - userid: - type: string - description: The Matrix ID of the matched user. - protocol: - type: string - description: The third party protocol. - fields: - type: object - description: The third party network fields used to identify this user. - properties: - user: - type: string - description: An example field, in this case the username for a Gitter user. + properties: + userid: + type: string + description: The Matrix ID of the matched user. + protocol: + type: string + description: The third party protocol. + fields: + type: object + description: The third party network fields used to identify this user. + properties: + user: + type: string + description: An example field, in this case the username for a Gitter user. 401: description: |- The homeserver has not supplied credentials to the application service. diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 41b5df16..2b07484a 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -237,20 +237,20 @@ paths: items: type: object title: User - properties: - userid: - type: string - description: The Matrix ID of the matched user. - protocol: - type: string - description: The third party protocol. - fields: - type: object - description: The third party network fields used to identify this user. - properties: - user: - type: string - description: An example field, in this case the username for a Gitter user. + properties: + userid: + type: string + description: The Matrix ID of the matched user. + protocol: + type: string + description: The third party protocol. + fields: + type: object + description: The third party network fields used to identify this user. + properties: + user: + type: string + description: An example field, in this case the username for a Gitter user. 404: description: The Matrix ID was not found examples: From e6218b56abe447c3ecd4ed2c3afa975db32583d1 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 Jul 2018 11:59:10 +0100 Subject: [PATCH 138/308] Switch properties to schema --- api/application-service/application_service.yaml | 2 +- api/client-server/third_party_lookup.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index 5edb1012..8c49043e 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -350,7 +350,7 @@ paths: items: type: object title: User - properties: + schema: userid: type: string description: The Matrix ID of the matched user. diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 2b07484a..0cc18ecd 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -237,7 +237,7 @@ paths: items: type: object title: User - properties: + schema: userid: type: string description: The Matrix ID of the matched user. From 8e22e9e85ab9da2e90b957f98c3155865fda8517 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 Jul 2018 12:00:20 +0100 Subject: [PATCH 139/308] Fix title underline length --- specification/third_party_lookup.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/third_party_lookup.rst b/specification/third_party_lookup.rst index 1279f9fe..6325dca0 100644 --- a/specification/third_party_lookup.rst +++ b/specification/third_party_lookup.rst @@ -10,7 +10,7 @@ Matrix client can use to interact with an application service through the facilitation of the homeserver. Third Party Lookups ------------------- +------------------- Application services can provide access to third party networks via bridging. This allows Matrix users to communicate with users on other communication From 4c38012122b2c6efc4943f7da55d4f3b3be3ae8e Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 Jul 2018 12:58:42 +0100 Subject: [PATCH 140/308] I can spell --- api/application-service/application_service.yaml | 2 +- api/client-server/third_party_lookup.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index 8c49043e..4951a29f 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -550,7 +550,7 @@ paths: description: |- Retreive an array of third party users from a Matrix ID. operationId: queryUserByID - paramters: + parameters: - in: query name: userid type: string diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 0cc18ecd..343cf2b7 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -305,7 +305,7 @@ paths: description: |- Retreive an array of third party users from a Matrix ID. operationId: queryUserByID - paramters: + parameters: - in: path name: userid type: string From 16860ddd22b8364ea501bbcf0034c5ae189b3bf3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 5 Jul 2018 10:46:53 -0600 Subject: [PATCH 141/308] De-quote --- api/client-server/search.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/search.yaml b/api/client-server/search.yaml index f7c96180..687140b3 100644 --- a/api/client-server/search.yaml +++ b/api/client-server/search.yaml @@ -103,7 +103,7 @@ paths: The order in which to search for results. By default, this is ``"rank"``. event_context: - title: "Include Event Context" + title: Include Event Context type: object description: |- Configures whether any context for the events From e81e0328c22aa2c0cb64b41791d692bb352d7683 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 5 Jul 2018 17:50:34 +0100 Subject: [PATCH 142/308] Fix single protocol metadata endpoint path, remove 500s --- .../application_service.yaml | 58 +------------------ 1 file changed, 1 insertion(+), 57 deletions(-) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index 4951a29f..86411ffa 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -92,13 +92,6 @@ paths: } schema: type: object - 500: - description: There was a problem completing the request. - examples: - application/json: { - } - schema: - type: object "/rooms/{roomAlias}": get: summary: Query if a room alias should exist on the application service. @@ -158,13 +151,6 @@ paths: } schema: type: object - 500: - description: There was a problem completing the request. - examples: - application/json: { - } - schema: - type: object "/users/{userId}": get: summary: Query if a user should exist on the application service. @@ -221,14 +207,7 @@ paths: } schema: type: object - 500: - description: There was a problem completing the request. - examples: - application/json: { - } - schema: - type: object - "/_matrix/app/unstable/thirdparty/{protocol}": + "/_matrix/app/unstable/thirdparty/protocol/{protocol}": get: summary: Retrieve metadata about a specific protocol that the application service supports. description: |- @@ -305,13 +284,6 @@ paths: } schema: type: object - 500: - description: There was a problem completing the request. - examples: - application/json: { - } - schema: - type: object "/_matrix/app/unstable/thirdparty/user/{protocol}": get: summary: Retrieve the Matrix ID of a corresponding third party user. @@ -391,13 +363,6 @@ paths: } schema: type: object - 500: - description: There was a problem completing the request. - examples: - application/json: { - } - schema: - type: object "/_matrix/app/unstable/thirdparty/location/{protocol}": get: summary: Retreive Matrix-side portal rooms leading to a third party location. @@ -471,13 +436,6 @@ paths: } schema: type: object - 500: - description: There was a problem completing the request. - examples: - application/json: { - } - schema: - type: object "/_matrix/app/unstable/thirdparty/location": get: summary: Reverse-lookup third party locations given a Matrix room alias. @@ -537,13 +495,6 @@ paths: } schema: type: object - 500: - description: There was a problem completing the request. - examples: - application/json: { - } - schema: - type: object "/_matrix/app/unstable/thirdparty/user": get: summary: Reverse-lookup third party users given a Matrix ID. @@ -598,12 +549,5 @@ paths: application/json: { "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } - schema: - type: object - 500: - description: There was a problem completing the request. - examples: - application/json: { - } schema: type: object \ No newline at end of file From f9a0a49e96a98eb0d96b2ef424868858235ab007 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 5 Jul 2018 20:22:59 +0100 Subject: [PATCH 143/308] Add a bunch more people to the speculator trusted list --- scripts/speculator/main.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/speculator/main.go b/scripts/speculator/main.go index 5e122503..12ec2aec 100644 --- a/scripts/speculator/main.go +++ b/scripts/speculator/main.go @@ -701,8 +701,8 @@ func ignoreExitCodeOne(err error) error { func main() { flag.Parse() - // It would be great to read this from github, but there's no convenient way to do so. - // Most of these memberships are "private", so would require some kind of auth. + // It would be great to read this from github + // cf https://github.com/matrix-org/matrix-doc/issues/1384 allowedMembers = map[string]bool{ "dbkr": true, "erikjohnston": true, @@ -715,6 +715,11 @@ func main() { "rxl881": true, "uhoreg": true, "turt2live": true, + "Half-Shot": true, + "anoadragon453": true, + "mujx": true, + "benparsons": true, + "KitsuneRal": true, } if err := initCache(); err != nil { log.Fatal(err) From bbf63d116cfca00ea9947547db5c84830736e94d Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 5 Jul 2018 16:55:11 -0600 Subject: [PATCH 144/308] Fix reference to error.yaml in forget room API --- api/client-server/leaving.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/leaving.yaml b/api/client-server/leaving.yaml index 7d8e9c89..513b5b4d 100644 --- a/api/client-server/leaving.yaml +++ b/api/client-server/leaving.yaml @@ -107,7 +107,7 @@ paths: "error": "User @example:matrix.org is in room !au1ba7o:matrix.org" } schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/error.yaml" 429: description: This request was rate-limited. schema: From d1c8ec9e489134a804834a049cd9f80794b48abb Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Thu, 5 Jul 2018 21:18:28 -0600 Subject: [PATCH 145/308] add circle config --- .circleci/config.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..e8df5956 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,19 @@ +version: 2 +jobs: + build-docs: + docker: + - image: uhoreg/matrix-doc-build + steps: + - checkout + - run: python scripts/gendoc.py + +workflows: + version: 2 + + egg-info: + jobs: + - build-docs + +notify: + webhooks: + - url: https://giles.cadair.com/circleci From 86b00451953d96e7af2eadb1c013415a488fc8c4 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Thu, 5 Jul 2018 21:25:15 -0600 Subject: [PATCH 146/308] virtualenvs are the worst --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e8df5956..c11e36ba 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,9 +2,10 @@ version: 2 jobs: build-docs: docker: - - image: uhoreg/matrix-doc-build + - image: circleci/python:2.7 steps: - checkout + - run: pip install -r scripts/requirements.txt - run: python scripts/gendoc.py workflows: From fe43b79e73f84708ad7ae5b641f7a69be8020170 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Thu, 5 Jul 2018 21:29:12 -0600 Subject: [PATCH 147/308] what's even the point --- .circleci/config.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c11e36ba..236a1d81 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,12 +1,17 @@ +gendoc: &gendoc + name: Generate the docs + command: | + source /env/bin/activate + scripts/gendoc.py + version: 2 jobs: build-docs: docker: - - image: circleci/python:2.7 + - image: uhoreg/matrix-doc-build steps: - checkout - - run: pip install -r scripts/requirements.txt - - run: python scripts/gendoc.py + - run: *gendoc workflows: version: 2 From b7c3a7402282eb1b9787d07158e27c93521dec9e Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Thu, 5 Jul 2018 21:31:57 -0600 Subject: [PATCH 148/308] artifacts --- .circleci/config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 236a1d81..06696633 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,6 +12,11 @@ jobs: steps: - checkout - run: *gendoc + - store_artifacts: + path: scripts/gen + - run: + name: "Doc build is available at:" + command: DOCS_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/scripts/gen/index.html"; echo $DOCS_URL workflows: version: 2 From be5232914a704808f74ac9a5f32b1591fc7988ff Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Thu, 5 Jul 2018 21:39:45 -0600 Subject: [PATCH 149/308] Maybe a swagger --- .circleci/config.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 06696633..c11eb70d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,6 +4,12 @@ gendoc: &gendoc source /env/bin/activate scripts/gendoc.py +gendoc: &genswagger + name: Generate the swagger + command: | + source /env/bin/activate + scripts/dump-swagger.py + version: 2 jobs: build-docs: @@ -18,12 +24,25 @@ jobs: name: "Doc build is available at:" command: DOCS_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/scripts/gen/index.html"; echo $DOCS_URL + build-swagger: + docker: + - image: uhoreg/matrix-doc-build + steps: + - checkout + - run: *genswagger + - store_artifacts: + path: scripts/swagger/api-docs.json + - run: + name: "Swagger UI is available at:" + command: SWAGGER_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/scripts/swagger/api-docs.json"; echo "https://matrix.org/docs/api/client-server/?url="$SWAGGER_URL + workflows: version: 2 - egg-info: + build-spec: jobs: - build-docs + - build-swagger notify: webhooks: From d5ce87e9567a38c9fbaafa2cbbfd90304794053b Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 6 Jul 2018 10:27:03 +0100 Subject: [PATCH 150/308] Split summary up into description and summary --- api/client-server/content-repo.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 0f4ef2da..3340808c 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -272,14 +272,14 @@ paths: "/config": get: summary: Get the configuration for the content repository. - Clients SHOULD use this as a guide when using content repository endpoints. - All values are intentionally left optional. Clients SHOULD follow - the advice given in the field description when the field is not available. + description: Clients SHOULD use this as a guide when using content repository endpoints. + All values are intentionally left optional. Clients SHOULD follow + the advice given in the field description when the field is not available. - **NOTE:** Both clients and server administrators should be aware that proxies - between the client and the server may affect the apparent behaviour of content - repository APIs, for example, proxies may enforce a lower upload size limit - than is advertised by the server on this endpoint. + **NOTE:** Both clients and server administrators should be aware that proxies + between the client and the server may affect the apparent behaviour of content + repository APIs, for example, proxies may enforce a lower upload size limit + than is advertised by the server on this endpoint. operationId: getConfig produces: ["application/json"] security: From 2e6cc8045cc3f0942b9c23caab1edd25f52e4915 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 6 Jul 2018 10:29:05 +0100 Subject: [PATCH 151/308] Better indent of description --- api/client-server/content-repo.yaml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 3340808c..1f1d6abc 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -272,14 +272,15 @@ paths: "/config": get: summary: Get the configuration for the content repository. - description: Clients SHOULD use this as a guide when using content repository endpoints. - All values are intentionally left optional. Clients SHOULD follow - the advice given in the field description when the field is not available. + description: |- + Clients SHOULD use this as a guide when using content repository endpoints. + All values are intentionally left optional. Clients SHOULD follow + the advice given in the field description when the field is not available. - **NOTE:** Both clients and server administrators should be aware that proxies - between the client and the server may affect the apparent behaviour of content - repository APIs, for example, proxies may enforce a lower upload size limit - than is advertised by the server on this endpoint. + **NOTE:** Both clients and server administrators should be aware that proxies + between the client and the server may affect the apparent behaviour of content + repository APIs, for example, proxies may enforce a lower upload size limit + than is advertised by the server on this endpoint. operationId: getConfig produces: ["application/json"] security: From 77f4ac54aaddee927812a6c3e10d844d8df00ef7 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 6 Jul 2018 10:36:17 +0100 Subject: [PATCH 152/308] Wordy intro sentence for description --- api/client-server/content-repo.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 1f1d6abc..d88961b7 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -273,6 +273,8 @@ paths: get: summary: Get the configuration for the content repository. description: |- + This endpoint allows clients to retrieve the configuration of the content + repository, such as upload limitations. Clients SHOULD use this as a guide when using content repository endpoints. All values are intentionally left optional. Clients SHOULD follow the advice given in the field description when the field is not available. From 516000bdc3888dce6844bebd066961ab5f50916e Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 08:53:04 -0600 Subject: [PATCH 153/308] Fix error reference in media repo --- api/client-server/content-repo.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/content-repo.yaml b/api/client-server/content-repo.yaml index 1397c8ee..b3e9517b 100644 --- a/api/client-server/content-repo.yaml +++ b/api/client-server/content-repo.yaml @@ -336,7 +336,7 @@ paths: 429: description: This request was rate-limited. schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/error.yaml" tags: - Media From f54d5a40398c3c6a1c252b01d38e50a850685dee Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 15:19:14 -0600 Subject: [PATCH 154/308] Convert build scripts to python 3 --- .gitignore | 2 +- README.rst | 2 +- api/check_examples.py | 6 +-- scripts/dump-swagger.py | 8 +-- scripts/gendoc.py | 38 +++++++------- scripts/proposals.py | 2 +- scripts/swagger-http-server.py | 12 ++--- scripts/templating/batesian/__init__.py | 7 ++- scripts/templating/batesian/sections.py | 8 +-- scripts/templating/batesian/units.py | 4 +- scripts/templating/build.py | 23 ++++----- .../templating/matrix_templates/__init__.py | 4 +- .../templating/matrix_templates/sections.py | 4 +- scripts/templating/matrix_templates/units.py | 49 ++++++++++--------- 14 files changed, 86 insertions(+), 83 deletions(-) diff --git a/.gitignore b/.gitignore index 84ac4951..58b6099e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ /api/node_modules /assets /assets.tar.gz -/env +/env* /scripts/gen /scripts/continuserv/continuserv /scripts/speculator/speculator diff --git a/README.rst b/README.rst index 6c87201c..c0949d9e 100644 --- a/README.rst +++ b/README.rst @@ -41,7 +41,7 @@ specs and event schemas in this repository. Preparation ----------- -To use the scripts, it is best to create a Python 2.x virtualenv as follows:: +To use the scripts, it is best to create a Python 3.5+ virtualenv as follows:: virtualenv env env/bin/pip install -r scripts/requirements.txt diff --git a/api/check_examples.py b/api/check_examples.py index be0676bb..3ecd5a4c 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -88,9 +88,9 @@ def check_swagger_file(filepath): with open(filepath) as f: swagger = yaml.load(f) - for path, path_api in swagger.get('paths', {}).items(): + for path, path_api in list(swagger.get('paths', {}).items()): - for method, request_api in path_api.items(): + for method, request_api in list(path_api.items()): request = "%s %s" % (method.upper(), path) for parameter in request_api.get('parameters', ()): if parameter['in'] == 'body': @@ -100,7 +100,7 @@ def check_swagger_file(filepath): responses = request_api['responses'] except KeyError: raise ValueError("No responses for %r" % (request,)) - for code, response in responses.items(): + for code, response in list(responses.items()): check_response(filepath, request, code, response) diff --git a/scripts/dump-swagger.py b/scripts/dump-swagger.py index 110c4446..6dd9864f 100755 --- a/scripts/dump-swagger.py +++ b/scripts/dump-swagger.py @@ -94,16 +94,16 @@ for filename in os.listdir(cs_api_dir): api = units.resolve_references(filepath, api) basePath = api['basePath'] - for path, methods in api["paths"].items(): + for path, methods in list(api["paths"].items()): path = (basePath + path).replace('%CLIENT_MAJOR_VERSION%', major_version) - for method, spec in methods.items(): - if "tags" in spec.keys(): + for method, spec in list(methods.items()): + if "tags" in list(spec.keys()): if path not in output["paths"]: output["paths"][path] = {} output["paths"][path][method] = spec -print "Generating %s" % output_file +print("Generating %s" % output_file) try: os.makedirs(os.path.dirname(output_file)) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 2b35f801..0c548611 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -31,6 +31,7 @@ script_dir = os.path.dirname(os.path.abspath(__file__)) docs_dir = os.path.dirname(script_dir) spec_dir = os.path.join(docs_dir, "specification") tmp_dir = os.path.join(script_dir, "tmp") +changelog_dir = os.path.join(docs_dir, "changelogs") VERBOSE = False @@ -151,7 +152,7 @@ def is_title_line(prev_line, line, title_styles): def get_rst(file_info, title_level, title_styles, spec_dir, adjust_titles): # string are file paths to RST blobs - if isinstance(file_info, basestring): + if isinstance(file_info, str): log("%s %s" % (">" * (1 + title_level), file_info)) with open(os.path.join(spec_dir, file_info), "r") as f: rst = None @@ -194,7 +195,7 @@ def build_spec(target, out_filename): spec_dir=spec_dir, adjust_titles=True ) - outfile.write(section) + outfile.write(section.encode('UTF-8')) """ @@ -279,15 +280,16 @@ def rst2html(i, o, stylesheets): def addAnchors(path): log("add anchors %s" % path) - with open(path, "r") as f: + with open(path, "rb") as f: lines = f.readlines() replacement = replacement = r'

\n\1' - with open(path, "w") as f: + with open(path, "wb") as f: for line in lines: + line = line.decode("UTF-8") line = re.sub(r'()', replacement, line.rstrip()) line = re.sub(r'(
)', replacement, line.rstrip()) - f.write(line + "\n") + f.write((line + "\n").encode('UTF-8')) def run_through_template(input_files, set_verbose, substitutions): @@ -297,7 +299,7 @@ def run_through_template(input_files, set_verbose, substitutions): "-i", "matrix_templates", ] - for k, v in substitutions.items(): + for k, v in list(substitutions.items()): args.append("--substitution=%s=%s" % (k, v)) if set_verbose: @@ -357,14 +359,14 @@ def get_build_target(all_targets, target_name): for i, entry in enumerate(group): if isinstance(entry, dict): group[i] = { - (rel_depth + depth): v for (rel_depth, v) in entry.items() + (rel_depth + depth): v for (rel_depth, v) in list(entry.items()) } return group resolved_files = [] for file_entry in target["files"]: # file_entry is a group id - if isinstance(file_entry, basestring) and file_entry.startswith("group:"): + if isinstance(file_entry, str) and file_entry.startswith("group:"): group = get_group(file_entry, 0) # The group may be resolved to a list of file entries, in which case # we want to extend the array to insert each of them rather than @@ -376,8 +378,8 @@ def get_build_target(all_targets, target_name): # file_entry is a dict which has more file entries as values elif isinstance(file_entry, dict): resolved_entry = {} - for (depth, entry) in file_entry.iteritems(): - if not isinstance(entry, basestring): + for (depth, entry) in list(file_entry.items()): + if not isinstance(entry, str): raise Exception( "Double-nested depths are not supported. Entry: %s" % (file_entry,) ) @@ -395,11 +397,11 @@ def get_build_target(all_targets, target_name): return build_target def log(line): - print "gendoc: %s" % line + print("gendoc: %s" % line) def logv(line): if VERBOSE: - print "gendoc:V: %s" % line + print("gendoc:V: %s" % line) def cleanup_env(): @@ -427,7 +429,7 @@ def main(targets, dest_dir, keep_intermediates, substitutions): target_defs = yaml.load(targ_file.read()) if targets == ["all"]: - targets = target_defs["targets"].keys() + targets = list(target_defs["targets"].keys()) log("Building spec [targets=%s]" % targets) @@ -441,17 +443,17 @@ def main(targets, dest_dir, keep_intermediates, substitutions): templated_files[target_name] = templated_file # we do all the templating at once, because it's slow - run_through_template(templated_files.values(), VERBOSE, substitutions) + run_through_template(list(templated_files.values()), VERBOSE, substitutions) stylesheets = glob.glob(os.path.join(script_dir, "css", "*.css")) - for target_name, templated_file in templated_files.iteritems(): + for target_name, templated_file in list(templated_files.items()): target = target_defs["targets"].get(target_name) version_label = None if target: version_label = target.get("version_label") if version_label: - for old, new in substitutions.items(): + for old, new in list(substitutions.items()): version_label = version_label.replace(old, new) rst_file = os.path.join(tmp_dir, "spec_%s.rst" % (target_name,)) @@ -479,8 +481,8 @@ def main(targets, dest_dir, keep_intermediates, substitutions): def list_targets(): with open(os.path.join(spec_dir, "targets.yaml"), "r") as targ_file: target_defs = yaml.load(targ_file.read()) - targets = target_defs["targets"].keys() - print "\n".join(targets) + targets = list(target_defs["targets"].keys()) + print("\n".join(targets)) def extract_major(s): diff --git a/scripts/proposals.py b/scripts/proposals.py index b489f0d2..156d5351 100755 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -14,7 +14,7 @@ prs = set() def getpage(url, page): resp = requests.get(url + str(page)) - for link in resp.links.values(): + for link in list(resp.links.values()): if link['rel'] == 'last': pagecount = re.search('page=(.+?)', link['url']).group(1) diff --git a/scripts/swagger-http-server.py b/scripts/swagger-http-server.py index 5ec00101..06d764aa 100755 --- a/scripts/swagger-http-server.py +++ b/scripts/swagger-http-server.py @@ -19,14 +19,14 @@ import argparse import os -import SimpleHTTPServer -import SocketServer +import http.server +import socketserver # Thanks to http://stackoverflow.com/a/13354482 -class MyHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): +class MyHTTPRequestHandler(http.server.SimpleHTTPRequestHandler): def end_headers(self): self.send_my_headers() - SimpleHTTPServer.SimpleHTTPRequestHandler.end_headers(self) + http.server.SimpleHTTPRequestHandler.end_headers(self) def send_my_headers(self): self.send_header("Access-Control-Allow-Origin", "*") @@ -49,7 +49,7 @@ if __name__ == '__main__': os.chdir(args.swagger_dir) - httpd = SocketServer.TCPServer(("localhost", args.port), + httpd = socketserver.TCPServer(("localhost", args.port), MyHTTPRequestHandler) - print "Serving at http://localhost:%i/api-docs.json" % args.port + print("Serving at http://localhost:%i/api-docs.json" % args.port) httpd.serve_forever() diff --git a/scripts/templating/batesian/__init__.py b/scripts/templating/batesian/__init__.py index da41b31b..f5d83a1e 100644 --- a/scripts/templating/batesian/__init__.py +++ b/scripts/templating/batesian/__init__.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from sets import Set class AccessKeyStore(object): @@ -22,10 +21,10 @@ class AccessKeyStore(object): if not existing_data: existing_data = {} self.data = existing_data - self.accessed_set = Set() + self.accessed_set = set() def keys(self): - return self.data.keys() + return list(self.data.keys()) def add(self, key, unit_dict): self.data[key] = unit_dict @@ -35,5 +34,5 @@ class AccessKeyStore(object): return self.data[key] def get_unaccessed_set(self): - data_list = Set(self.data.keys()) + data_list = set(self.data.keys()) return data_list - self.accessed_set \ No newline at end of file diff --git a/scripts/templating/batesian/sections.py b/scripts/templating/batesian/sections.py index c541d771..2e5593d6 100644 --- a/scripts/templating/batesian/sections.py +++ b/scripts/templating/batesian/sections.py @@ -29,7 +29,7 @@ class Sections(object): def log(self, text): if self.debug: - print "batesian:sections: %s" % text + print("batesian:sections: %s" % text) def get_sections(self): render_list = inspect.getmembers(self, predicate=inspect.ismethod) @@ -40,7 +40,7 @@ class Sections(object): section_key = func_name[len("render_"):] self.log("Generating section '%s'" % section_key) section = func() - if isinstance(section, basestring): + if isinstance(section, str): if section_key in section_dict: raise Exception( ("%s : Section %s already exists. It must have been " + @@ -54,8 +54,8 @@ class Sections(object): ) elif isinstance(section, dict): self.log(" Generated multiple sections:") - for (k, v) in section.iteritems(): - if not isinstance(k, basestring) or not isinstance(v, basestring): + for (k, v) in list(section.items()): + if not isinstance(k, str) or not isinstance(v, str): raise Exception( ("Method %s returned multiple sections as a dict but " + "expected the dict elements to be strings but they aren't.") % diff --git a/scripts/templating/batesian/units.py b/scripts/templating/batesian/units.py index 8f748f6d..82cc52f9 100644 --- a/scripts/templating/batesian/units.py +++ b/scripts/templating/batesian/units.py @@ -41,7 +41,7 @@ class Units(object): trace = inspect.stack() if len(trace) > 1 and len(trace[1]) > 2: func_name = trace[1][3] + ":" - print "batesian:units:%s %s" % (func_name, text) + print("batesian:units:%s %s" % (func_name, text)) def get_units(self, debug=False): unit_list = inspect.getmembers(self, predicate=inspect.ismethod) @@ -50,7 +50,7 @@ class Units(object): if not func_name.startswith("load_"): continue unit_key = func_name[len("load_"):] - if len(inspect.getargs(func.func_code).args) > 1: + if len(inspect.getargs(func.__code__).args) > 1: unit_dict[unit_key] = func(self.substitutions) else: unit_dict[unit_key] = func() diff --git a/scripts/templating/build.py b/scripts/templating/build.py index d18569b6..de9ead35 100755 --- a/scripts/templating/build.py +++ b/scripts/templating/build.py @@ -63,6 +63,7 @@ import sys from textwrap import TextWrapper from matrix_templates.units import TypeTableRow +from functools import reduce def create_from_template(template, sections): @@ -138,7 +139,7 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={} return reduce(max, rowwidths, default if default is not None else default_width) - results = map(colwidth, keys, defaults) + results = list(map(colwidth, keys, defaults)) return results # make Jinja aware of the templates and filters @@ -167,16 +168,16 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={} # print out valid section keys if no file supplied if not files: - print "\nValid template variables:" - for key in sections.keys(): + print("\nValid template variables:") + for key in list(sections.keys()): sec_text = "" if (len(sections[key]) > 75) else ( "(Value: '%s')" % sections[key] ) sec_info = "%s characters" % len(sections[key]) if sections[key].count("\n") > 0: sec_info += ", %s lines" % sections[key].count("\n") - print " %s" % key - print " %s %s" % (sec_info, sec_text) + print(" %s" % key) + print(" %s %s" % (sec_info, sec_text)) return # check the input files and substitute in sections where required @@ -190,8 +191,8 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={} def process_file(env, sections, filename, output_filename): log("Parsing input template: %s" % filename) - with open(filename, "r") as file_stream: - temp_str = file_stream.read().decode("utf-8") + with open(filename, "rb") as file_stream: + temp_str = file_stream.read().decode('UTF-8') # do sanity checking on the template to make sure they aren't reffing things # which will never be replaced with a section. @@ -210,16 +211,16 @@ def process_file(env, sections, filename, output_filename): # Do these substitutions outside of the ordinary templating system because # we want them to apply to things like the underlying swagger used to # generate the templates, not just the top-level sections. - for old, new in substitutions.items(): + for old, new in list(substitutions.items()): output = output.replace(old, new) - with open(output_filename, "w") as f: - f.write(output.encode("utf-8")) + with open(output_filename, "wb") as f: + f.write(output.encode('UTF-8')) log("Output file for: %s" % output_filename) def log(line): - print "batesian: %s" % line + print("batesian: %s" % line) if __name__ == '__main__': parser = ArgumentParser( diff --git a/scripts/templating/matrix_templates/__init__.py b/scripts/templating/matrix_templates/__init__.py index 6b46192c..b81c5a30 100644 --- a/scripts/templating/matrix_templates/__init__.py +++ b/scripts/templating/matrix_templates/__init__.py @@ -11,8 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from sections import MatrixSections -from units import MatrixUnits +from .sections import MatrixSections +from .units import MatrixUnits import os exports = { diff --git a/scripts/templating/matrix_templates/sections.py b/scripts/templating/matrix_templates/sections.py index 64e32aa4..71c1acf3 100644 --- a/scripts/templating/matrix_templates/sections.py +++ b/scripts/templating/matrix_templates/sections.py @@ -86,7 +86,7 @@ class MatrixSections(Sections): # the key is the section name and the value is the value of the section def render_group_http_apis(self): # map all swagger_apis to the form $GROUP_http_api - swagger_groups = self.units.get("swagger_apis").keys() + swagger_groups = list(self.units.get("swagger_apis").keys()) renders = {} for group in swagger_groups: sortFnOrPathList = None @@ -134,7 +134,7 @@ class MatrixSections(Sections): "m.room.message#m.file" ] other_msgtypes = [ - k for k in schemas.keys() if k.startswith("m.room.message#") and + k for k in list(schemas.keys()) if k.startswith("m.room.message#") and k not in msgtype_order ] for event_name in (msgtype_order + other_msgtypes): diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index dac183d1..40fd96a2 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -29,11 +29,12 @@ import os.path import re import subprocess import sys -import urllib +import urllib.request, urllib.parse, urllib.error import yaml +from functools import reduce matrix_doc_dir=reduce(lambda acc,_: os.path.dirname(acc), - range(1, 5), os.path.abspath(__file__)) + list(range(1, 5)), os.path.abspath(__file__)) HTTP_APIS = { os.path.join(matrix_doc_dir, "api/application-service"): "as", @@ -124,7 +125,7 @@ def resolve_references(path, schema): else: result = OrderedDict() - for key, value in schema.items(): + for key, value in list(schema.items()): result[key] = resolve_references(path, value) return result elif isinstance(schema, list): @@ -147,7 +148,7 @@ def inherit_parents(obj): # settings defined in the child take priority over the parents, so we # iterate through the parents first, and then overwrite with the settings # from the child. - for p in map(inherit_parents, parents) + [obj]: + for p in list(map(inherit_parents, parents)) + [obj]: # child blats out type, title and description for key in ('type', 'title', 'description'): if p.get(key): @@ -209,7 +210,7 @@ def get_json_schema_object_fields(obj, enforce_title=False): props = obj.get("patternProperties") if props: # try to replace horrible regex key names with pretty x-pattern ones - for key_name in props.keys(): + for key_name in list(props.keys()): pretty_key = props[key_name].get("x-pattern") if pretty_key: props[pretty_key] = props[key_name] @@ -250,12 +251,12 @@ def get_json_schema_object_fields(obj, enforce_title=False): tables.extend(res["tables"]) logger.debug("Done property %s" % key_name) - except Exception, e: + except Exception as e: e2 = Exception("Error reading property %s.%s: %s" % (obj_title, key_name, str(e))) # throw the new exception with the old stack trace, so that # we don't lose information about where the error occurred. - raise e2, None, sys.exc_info()[2] + raise e2.with_traceback(sys.exc_info()[2]) tables.insert(0, TypeTable(title=obj_title, rows=first_table_rows)) @@ -380,7 +381,7 @@ def get_example_for_schema(schema): if 'properties' not in schema: raise Exception('"object" property has neither properties nor example') res = OrderedDict() - for prop_name, prop in schema['properties'].iteritems(): + for prop_name, prop in list(schema['properties'].items()): logger.debug("Parsing property %r" % prop_name) prop_example = get_example_for_schema(prop) res[prop_name] = prop_example @@ -523,7 +524,7 @@ class MatrixUnits(Units): if param_loc == "path": path_template = path_template.replace( - "{%s}" % param_name, urllib.quote(example) + "{%s}" % param_name, urllib.parse.quote(example) ) elif param_loc == "query": if type(example) == list: @@ -532,7 +533,7 @@ class MatrixUnits(Units): else: example_query_params.append((param_name, example)) - except Exception, e: + except Exception as e: raise Exception("Error handling parameter %s" % param_name, e) # endfor[param] good_response = None @@ -556,14 +557,14 @@ class MatrixUnits(Units): ) if "headers" in good_response: headers = TypeTable() - for (header_name, header) in good_response["headers"].iteritems(): + for (header_name, header) in list(good_response["headers"].items()): headers.add_row( TypeTableRow(key=header_name, title=header["type"], desc=header["description"]), ) endpoint["res_headers"] = headers query_string = "" if len( - example_query_params) == 0 else "?" + urllib.urlencode( + example_query_params) == 0 else "?" + urllib.parse.urlencode( example_query_params) if example_body: endpoint["example"][ @@ -605,17 +606,17 @@ class MatrixUnits(Units): body_tables = req_body_tables[1:] endpoint_data['req_body_tables'].extend(body_tables) - except Exception, e: + except Exception as e: e2 = Exception( "Error decoding body of API endpoint %s %s: %s" % (endpoint_data["method"], endpoint_data["path"], e) ) - raise e2, None, sys.exc_info()[2] + raise e2.with_traceback(sys.exc_info()[2]) def load_swagger_apis(self): apis = {} - for path, suffix in HTTP_APIS.items(): + for path, suffix in list(HTTP_APIS.items()): for filename in os.listdir(path): if not filename.endswith(".yaml"): continue @@ -711,12 +712,12 @@ class MatrixUnits(Units): if filename != event_name: examples[event_name] = examples.get(event_name, []) examples[event_name].append(example) - except Exception, e: + except Exception as e: e2 = Exception("Error reading event example "+filepath+": "+ str(e)) # throw the new exception with the old stack trace, so that # we don't lose information about where the error occurred. - raise e2, None, sys.exc_info()[2] + raise e2.with_traceback(sys.exc_info()[2]) return examples @@ -730,12 +731,12 @@ class MatrixUnits(Units): filepath = os.path.join(path, filename) try: schemata[filename] = self.read_event_schema(filepath) - except Exception, e: + except Exception as e: e2 = Exception("Error reading event schema "+filepath+": "+ str(e)) # throw the new exception with the old stack trace, so that # we don't lose information about where the error occurred. - raise e2, None, sys.exc_info()[2] + raise e2.with_traceback(sys.exc_info()[2]) return schemata @@ -871,7 +872,7 @@ class MatrixUnits(Units): ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], stderr=null, cwd=cwd, - ).strip() + ).strip().decode('UTF-8') except subprocess.CalledProcessError: git_branch = "" try: @@ -879,7 +880,7 @@ class MatrixUnits(Units): ['git', 'describe', '--exact-match'], stderr=null, cwd=cwd, - ).strip() + ).strip().decode('UTF-8') git_tag = "tag=" + git_tag except subprocess.CalledProcessError: git_tag = "" @@ -888,7 +889,7 @@ class MatrixUnits(Units): ['git', 'rev-parse', '--short', 'HEAD'], stderr=null, cwd=cwd, - ).strip() + ).strip().decode('UTF-8') except subprocess.CalledProcessError: git_commit = "" try: @@ -897,7 +898,7 @@ class MatrixUnits(Units): ['git', 'describe', '--dirty=' + dirty_string, "--all"], stderr=null, cwd=cwd, - ).strip().endswith(dirty_string) + ).strip().decode('UTF-8').endswith(dirty_string) git_dirty = "dirty" if is_dirty else "" except subprocess.CalledProcessError: git_dirty = "" @@ -908,7 +909,7 @@ class MatrixUnits(Units): s for s in (git_branch, git_tag, git_commit, git_dirty,) if s - ).encode("ascii") + ).encode("ascii").decode('ascii') return { "string": git_version, "revision": git_commit From 9277e4c52d70472cd58d7b2bf0ec6329e8228b8a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 16:32:27 -0600 Subject: [PATCH 155/308] Add initial towncrier support --- .gitignore | 1 + changelogs/client_server/pyproject.toml | 30 ++++++++++++++++++++ scripts/requirements.txt | 1 + scripts/templating/matrix_templates/units.py | 23 +++++++++++++-- 4 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 changelogs/client_server/pyproject.toml diff --git a/.gitignore b/.gitignore index 58b6099e..a850d2fa 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ /templating/out *.pyc *.swp +_rendered.rst diff --git a/changelogs/client_server/pyproject.toml b/changelogs/client_server/pyproject.toml new file mode 100644 index 00000000..d5daface --- /dev/null +++ b/changelogs/client_server/pyproject.toml @@ -0,0 +1,30 @@ +[tool.towncrier] + filename = "_rendered.rst" + directory = "newsfragments" + issue_format = "`#{issue} `_" + title_format = "{version}" + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "new" + name = "New Endpoints" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Backwards Compatible Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "clarification" + name = "Spec Clarifications" + showcontent = true diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 37d99dd3..6f7951d6 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -7,3 +7,4 @@ Jinja2 >= 2.9.6 jsonschema >= 2.6.0 PyYAML >= 3.12 requests >= 2.18.4 +towncrier == 18.6.0rc1 diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 40fd96a2..0dbf8104 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -832,12 +832,28 @@ class MatrixUnits(Units): path = os.path.join(CHANGELOG_DIR, f) name = f[:-4] + # If there's a directory with the same name, we'll try to generate + # a towncrier changelog and prepend it to the general changelog. + tc_path = os.path.join(CHANGELOG_DIR, name) + tc_lines = [] + if os.path.isdir(tc_path): + logger.info("Generating towncrier changelog for: %s" % name) + try: + raw_log = subprocess.check_output( + ['towncrier', '--version', 'Unreleased Changes', '--name', name, '--draft'], + stderr=subprocess.PIPE, + cwd=tc_path, + ).strip().decode('UTF-8') + except subprocess.CalledProcessError: + raw_log = "" + tc_lines = raw_log.splitlines() + title_part = None changelog_lines = [] with open(path, "r") as f: lines = f.readlines() prev_line = None - for line in lines: + for line in (tc_lines + lines): if prev_line is None: prev_line = line continue @@ -853,7 +869,10 @@ class MatrixUnits(Units): # then bail out. changelog_lines.pop() break - changelog_lines.append(" " + line) + # Don't generate subheadings (we'll keep the title though) + if re.match("^[-]{3,}$", line.strip()): + continue + changelog_lines.append(" " + line + '\n') changelogs[name] = "".join(changelog_lines) return changelogs From ee9abc5fd76c4e8e62a9a737fbf4ea33afddead5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 16:41:54 -0600 Subject: [PATCH 156/308] Convert unstable changelog to towncrier --- changelogs/client_server.rst | 70 ------------------- .../client_server/newsfragments/1096.new | 1 + .../newsfragments/1097.clarification | 1 + .../client_server/newsfragments/1104.feature | 1 + .../newsfragments/1106.clarification | 1 + .../newsfragments/1109.clarification | 1 + .../client_server/newsfragments/1110.new | 1 + .../newsfragments/1137.clarification | 1 + .../newsfragments/1139.clarification | 1 + .../client_server/newsfragments/1141.feature | 1 + .../client_server/newsfragments/1142.feature | 1 + .../client_server/newsfragments/1151.feature | 1 + .../newsfragments/1152.clarification | 1 + .../newsfragments/1158.clarification | 3 + .../client_server/newsfragments/1158.feature | 1 + .../client_server/newsfragments/1239.new | 1 + .../newsfragments/1245.clarification | 1 + .../client_server/newsfragments/1263.feature | 1 + .../client_server/newsfragments/1264.feature | 1 + .../client_server/newsfragments/1265.feature | 1 + .../client_server/newsfragments/1274.feature | 1 + .../newsfragments/1329.clarification | 1 + .../client_server/newsfragments/1361.feature | 1 + .../newsfragments/1362.clarification | 1 + .../client_server/newsfragments/1364.feature | 1 + .../newsfragments/1378.clarification | 1 + .../newsfragments/1380.clarification | 1 + .../newsfragments/1381.clarification | 1 + 28 files changed, 29 insertions(+), 70 deletions(-) create mode 100644 changelogs/client_server/newsfragments/1096.new create mode 100644 changelogs/client_server/newsfragments/1097.clarification create mode 100644 changelogs/client_server/newsfragments/1104.feature create mode 100644 changelogs/client_server/newsfragments/1106.clarification create mode 100644 changelogs/client_server/newsfragments/1109.clarification create mode 100644 changelogs/client_server/newsfragments/1110.new create mode 100644 changelogs/client_server/newsfragments/1137.clarification create mode 100644 changelogs/client_server/newsfragments/1139.clarification create mode 100644 changelogs/client_server/newsfragments/1141.feature create mode 100644 changelogs/client_server/newsfragments/1142.feature create mode 100644 changelogs/client_server/newsfragments/1151.feature create mode 100644 changelogs/client_server/newsfragments/1152.clarification create mode 100644 changelogs/client_server/newsfragments/1158.clarification create mode 100644 changelogs/client_server/newsfragments/1158.feature create mode 100644 changelogs/client_server/newsfragments/1239.new create mode 100644 changelogs/client_server/newsfragments/1245.clarification create mode 100644 changelogs/client_server/newsfragments/1263.feature create mode 100644 changelogs/client_server/newsfragments/1264.feature create mode 100644 changelogs/client_server/newsfragments/1265.feature create mode 100644 changelogs/client_server/newsfragments/1274.feature create mode 100644 changelogs/client_server/newsfragments/1329.clarification create mode 100644 changelogs/client_server/newsfragments/1361.feature create mode 100644 changelogs/client_server/newsfragments/1362.clarification create mode 100644 changelogs/client_server/newsfragments/1364.feature create mode 100644 changelogs/client_server/newsfragments/1378.clarification create mode 100644 changelogs/client_server/newsfragments/1380.clarification create mode 100644 changelogs/client_server/newsfragments/1381.clarification diff --git a/changelogs/client_server.rst b/changelogs/client_server.rst index 32654025..d2ef7f2d 100644 --- a/changelogs/client_server.rst +++ b/changelogs/client_server.rst @@ -1,73 +1,3 @@ -Unreleased changes -================== - -- Changes to the API which will be backwards-compatible for clients: - - - New endpoints: - - - ``POST /user_directory/search`` - (`#1096 `_). - - ``GET /rooms/{roomId}/event/{eventId}`` - (`#1110 `_). - - ``POST /delete_devices`` - (`#1239 `_). - - - Sticker messages: - - Add sticker message event definition. - (`#1158 `_). - - Document the ``server_name`` parameter on ``/join/{roomIdOrAlias}`` - (`#1364 `_). - - Add 'token' parameter to /keys/query endpoint - (`#1104 `_). - - Add the room visibility options for the room directory - (`#1141 `_). - - Add spec for ignoring users - (`#1142 `_). - - Add the ``/register/available`` endpoint for username availability - (`#1151 `_). - - Add ``allow_remote`` to the content repo to avoid routing loops - (`#1265 `_). - - Add report content API - (`#1264 `_). - - Document ``/logout/all`` endpoint - (`#1263 `_). - - Document `highlights` field in /search response - (`#1274 `_). - - Document the GET version of ``/login`` - (`#1361 `_). - -- Spec clarifications: - - - Update ``ImageInfo`` and ``ThumbnailInfo`` dimension schema descriptions - to clarify that they relate to intended display size, as opposed to the - intrinsic size of the image file. - (`#1158 `_). - - Mark ``home_server`` return field for ``/login`` and ``/register`` - endpoints as deprecated - (`#1097 `_). - - Fix response format of ``/keys/changes`` endpoint - (`#1106 `_). - - Clarify default values for some fields on the /search API - (`#1109 `_). - - Fix the representation of ``m.presence`` events - (`#1137 `_). - - Clarify that ``m.tag`` ordering is done with numbers, not strings - (`#1139 `_). - - Clarify that ``/account/whoami`` should consider application services - (`#1152 `_). - - Mark ``GET /rooms/{roomId}/members`` as requiring authentication - (`#1245 `_). - - Define what a ``RoomEvent`` is on ``/rooms/{roomId}/messages`` - (`#1380 `_). - - Describe ``StateEvent`` for ``/createRoom`` - (`#1329 `_). - - Describe how the ``reason`` is handled for kicks/bans - (`#1362 `_). - - Clarify that clients must leave rooms before forgetting them - (`#1378 `_). - - Clarify the request and result types on ``/search`` - (`#1381 `_). - r0.3.0 ====== diff --git a/changelogs/client_server/newsfragments/1096.new b/changelogs/client_server/newsfragments/1096.new new file mode 100644 index 00000000..50d86879 --- /dev/null +++ b/changelogs/client_server/newsfragments/1096.new @@ -0,0 +1 @@ +``POST /user_directory/search`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1097.clarification b/changelogs/client_server/newsfragments/1097.clarification new file mode 100644 index 00000000..2a7cb93a --- /dev/null +++ b/changelogs/client_server/newsfragments/1097.clarification @@ -0,0 +1 @@ +Mark ``home_server`` return field for ``/login`` and ``/register`` endpoints as deprecated \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1104.feature b/changelogs/client_server/newsfragments/1104.feature new file mode 100644 index 00000000..9b85343f --- /dev/null +++ b/changelogs/client_server/newsfragments/1104.feature @@ -0,0 +1 @@ +Add ``token`` parameter to the ``/keys/query`` endpoint \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1106.clarification b/changelogs/client_server/newsfragments/1106.clarification new file mode 100644 index 00000000..f7a1fe3e --- /dev/null +++ b/changelogs/client_server/newsfragments/1106.clarification @@ -0,0 +1 @@ +Fix response format of ``/keys/changes`` endpoint \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1109.clarification b/changelogs/client_server/newsfragments/1109.clarification new file mode 100644 index 00000000..176d9403 --- /dev/null +++ b/changelogs/client_server/newsfragments/1109.clarification @@ -0,0 +1 @@ +Clarify default values for some fields on the ``/search`` API \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1110.new b/changelogs/client_server/newsfragments/1110.new new file mode 100644 index 00000000..e1b80b8b --- /dev/null +++ b/changelogs/client_server/newsfragments/1110.new @@ -0,0 +1 @@ +``GET /rooms/{roomId}/event/{eventId}`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1137.clarification b/changelogs/client_server/newsfragments/1137.clarification new file mode 100644 index 00000000..5ad8ec34 --- /dev/null +++ b/changelogs/client_server/newsfragments/1137.clarification @@ -0,0 +1 @@ +Fix the representation of ``m.presence`` events \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1139.clarification b/changelogs/client_server/newsfragments/1139.clarification new file mode 100644 index 00000000..b5193ad3 --- /dev/null +++ b/changelogs/client_server/newsfragments/1139.clarification @@ -0,0 +1 @@ +Clarify that ``m.tag`` ordering is done with numbers, not strings \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1141.feature b/changelogs/client_server/newsfragments/1141.feature new file mode 100644 index 00000000..da041819 --- /dev/null +++ b/changelogs/client_server/newsfragments/1141.feature @@ -0,0 +1 @@ +Add the room visibility options for the room directory \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1142.feature b/changelogs/client_server/newsfragments/1142.feature new file mode 100644 index 00000000..0a0842c4 --- /dev/null +++ b/changelogs/client_server/newsfragments/1142.feature @@ -0,0 +1 @@ +Add spec for ignoring users \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1151.feature b/changelogs/client_server/newsfragments/1151.feature new file mode 100644 index 00000000..8875812b --- /dev/null +++ b/changelogs/client_server/newsfragments/1151.feature @@ -0,0 +1 @@ +Add the ``/register/available`` endpoint for username availability \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1152.clarification b/changelogs/client_server/newsfragments/1152.clarification new file mode 100644 index 00000000..bbecc9b2 --- /dev/null +++ b/changelogs/client_server/newsfragments/1152.clarification @@ -0,0 +1 @@ +Clarify that ``/account/whoami`` should consider application services \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1158.clarification b/changelogs/client_server/newsfragments/1158.clarification new file mode 100644 index 00000000..dc1f6d14 --- /dev/null +++ b/changelogs/client_server/newsfragments/1158.clarification @@ -0,0 +1,3 @@ +Update ``ImageInfo`` and ``ThumbnailInfo`` dimension schema descriptions +to clarify that they relate to intended display size, as opposed to the +intrinsic size of the image file. \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1158.feature b/changelogs/client_server/newsfragments/1158.feature new file mode 100644 index 00000000..a55df4fb --- /dev/null +++ b/changelogs/client_server/newsfragments/1158.feature @@ -0,0 +1 @@ +Add sticker messages \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1239.new b/changelogs/client_server/newsfragments/1239.new new file mode 100644 index 00000000..9bcf357d --- /dev/null +++ b/changelogs/client_server/newsfragments/1239.new @@ -0,0 +1 @@ +``POST /delete_devices`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1245.clarification b/changelogs/client_server/newsfragments/1245.clarification new file mode 100644 index 00000000..e0a63834 --- /dev/null +++ b/changelogs/client_server/newsfragments/1245.clarification @@ -0,0 +1 @@ +Mark ``GET /rooms/{roomId}/members`` as requiring authentication \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1263.feature b/changelogs/client_server/newsfragments/1263.feature new file mode 100644 index 00000000..04964a7d --- /dev/null +++ b/changelogs/client_server/newsfragments/1263.feature @@ -0,0 +1 @@ +Document ``/logout/all`` endpoint \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1264.feature b/changelogs/client_server/newsfragments/1264.feature new file mode 100644 index 00000000..9cb06f71 --- /dev/null +++ b/changelogs/client_server/newsfragments/1264.feature @@ -0,0 +1 @@ +Add report content API \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1265.feature b/changelogs/client_server/newsfragments/1265.feature new file mode 100644 index 00000000..1e270fa8 --- /dev/null +++ b/changelogs/client_server/newsfragments/1265.feature @@ -0,0 +1 @@ +Add ``allow_remote`` to the content repo to avoid routing loops \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1274.feature b/changelogs/client_server/newsfragments/1274.feature new file mode 100644 index 00000000..d4958131 --- /dev/null +++ b/changelogs/client_server/newsfragments/1274.feature @@ -0,0 +1 @@ +Document `highlights` field in /search response \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1329.clarification b/changelogs/client_server/newsfragments/1329.clarification new file mode 100644 index 00000000..970d3d94 --- /dev/null +++ b/changelogs/client_server/newsfragments/1329.clarification @@ -0,0 +1 @@ +Describe ``StateEvent`` for ``/createRoom`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1361.feature b/changelogs/client_server/newsfragments/1361.feature new file mode 100644 index 00000000..b1d4c2f1 --- /dev/null +++ b/changelogs/client_server/newsfragments/1361.feature @@ -0,0 +1 @@ +Document the GET version of ``/login`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1362.clarification b/changelogs/client_server/newsfragments/1362.clarification new file mode 100644 index 00000000..1deb4500 --- /dev/null +++ b/changelogs/client_server/newsfragments/1362.clarification @@ -0,0 +1 @@ +Describe how the ``reason`` is handled for kicks/bans \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1364.feature b/changelogs/client_server/newsfragments/1364.feature new file mode 100644 index 00000000..733d6a1f --- /dev/null +++ b/changelogs/client_server/newsfragments/1364.feature @@ -0,0 +1 @@ +Document the ``server_name`` parameter on ``/join/{roomIdOrAlias}`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1378.clarification b/changelogs/client_server/newsfragments/1378.clarification new file mode 100644 index 00000000..f952428b --- /dev/null +++ b/changelogs/client_server/newsfragments/1378.clarification @@ -0,0 +1 @@ +Clarify that clients must leave rooms before forgetting them \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1380.clarification b/changelogs/client_server/newsfragments/1380.clarification new file mode 100644 index 00000000..490a9a43 --- /dev/null +++ b/changelogs/client_server/newsfragments/1380.clarification @@ -0,0 +1 @@ +Define what a ``RoomEvent`` is on ``/rooms/{roomId}/messages`` \ No newline at end of file diff --git a/changelogs/client_server/newsfragments/1381.clarification b/changelogs/client_server/newsfragments/1381.clarification new file mode 100644 index 00000000..e5e599ac --- /dev/null +++ b/changelogs/client_server/newsfragments/1381.clarification @@ -0,0 +1 @@ +Clarify the request and result types on ``/search`` \ No newline at end of file From c2f85788ded8970cad38bc261314ae148c492e20 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 16:54:14 -0600 Subject: [PATCH 157/308] Add documentation about how to add to the changelog --- CONTRIBUTING.rst | 35 +++++++++++++++++++++++++++++++++++ scripts/gendoc.py | 3 ++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 2a7416b1..c592cf02 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -69,6 +69,41 @@ For such changes, please do just open a `pull request`_. .. _pull request: https://help.github.com/articles/about-pull-requests + +Adding to the changelog +~~~~~~~~~~~~~~~~~~~~~~~ + +Currently only changes to the client-server API need to end up in a changelog. The +other APIs are not yet stable and therefore do not have a changelog. Adding to the +changelog can only be done after you've opened your pull request, so be sure to do +that first. + +The changelog is managed by Towncrier (https://github.com/hawkowl/towncrier) in the +form of "news fragments". The news fragments for the client-server API are stored +under ``changelogs/client_server/newsfragments``. + +To create a changelog entry, create a file named in the format ``prNumber.type`` in +the ``newsfragments`` directory. The ``type`` can be one of the following: + +* ``new`` - Used when adding new endpoints. Please have the file contents be the + method and route being added, surrounded in RST code tags. For example: ``POST + /accounts/whoami`` + +* ``feature`` - Used when adding backwards-compatible changes to the API. + +* ``clarification`` - Used when an area of the spec is being improved upon and does + not change or introduce any functionality. + +* ``breaking`` - Used when the change is not backwards compatible. + +* ``deprecation`` - Used when deprecating something + +All news fragments must have a brief summary explaining the change in the contents +of the file. + +Changes that do not change the spec, such as changes to the build script, formatting, +CSS, etc should not get a news fragment. + Sign off -------- diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 0c548611..31ee9e69 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -405,7 +405,8 @@ def logv(line): def cleanup_env(): - shutil.rmtree(tmp_dir) + #shutil.rmtree(tmp_dir) + pass def mkdirp(d) : From 34089e302c8415a34f69a200f4aee689fd97a0c8 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 17:06:08 -0600 Subject: [PATCH 158/308] Add general documentation for maintainers --- changelogs/README.md | 55 +++++++++++++++++++++++++ changelogs/client_server/pyproject.toml | 2 +- 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 changelogs/README.md diff --git a/changelogs/README.md b/changelogs/README.md new file mode 100644 index 00000000..10d69cb6 --- /dev/null +++ b/changelogs/README.md @@ -0,0 +1,55 @@ + + +# Changelogs + +[Towncrier](https://github.com/hawkowl/towncrier) is used to manage the changelog and +keep it up to date. Because of this, updating a changelog is really easy. + +## How to update a changelog when releasing an API + +1. Ensure you're in your Python 3 virtual environment +2. `cd` your way to the API you're releasing (eg: `cd changelogs/client_server`) +3. Run `towncrier --version "r0.4.0" --name "client-server" --yes` substituting the + variables as approprite. Note that `--name` is required although the value is ignored. +4. Commit the changes and finish the release process. + +## How to prepare a changelog for a new API + +For this example, we're going to pretend that the `server_server` API doesn't exist. + +1. Create the file `changelogs/server_server.rst` +2. Create the folder `changelogs/server_server` +3. In the new folder, create a `pyproject.toml` file with these contents: + ```toml + [tool.towncrier] + filename = "../server_server.rst" + directory = "newsfragments" + issue_format = "`#{issue} `_" + title_format = "{version}" + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "new" + name = "New Endpoints" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Backwards Compatible Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "clarification" + name = "Spec Clarifications" + showcontent = true + ``` +4. Use the changelog in whatever RST documents you need diff --git a/changelogs/client_server/pyproject.toml b/changelogs/client_server/pyproject.toml index d5daface..8fa3f6b5 100644 --- a/changelogs/client_server/pyproject.toml +++ b/changelogs/client_server/pyproject.toml @@ -1,5 +1,5 @@ [tool.towncrier] - filename = "_rendered.rst" + filename = "../client_server.rst" directory = "newsfragments" issue_format = "`#{issue} `_" title_format = "{version}" From 63b926a4aaae18293426cc10efa658a8644b7e42 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 17:19:04 -0600 Subject: [PATCH 159/308] Make Travis CI use python 3 --- .travis.yml | 4 ++++ scripts/test-and-build.sh | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7d84f237..98dd14b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,10 @@ sudo: false git: depth: 1 +before_install: +- sudo apt-get update +- sudo apt-get install python3 + # test-and-build does the installation, so tell travis to skip the # installation step install: true diff --git a/scripts/test-and-build.sh b/scripts/test-and-build.sh index 7794f826..e459a370 100755 --- a/scripts/test-and-build.sh +++ b/scripts/test-and-build.sh @@ -4,7 +4,7 @@ set -ex cd `dirname $0`/.. -virtualenv env +virtualenv -p python3 env . env/bin/activate pip install -r scripts/requirements.txt From f68c82bd6c3d8959004ee564ae9dce6769aa99a8 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 6 Jul 2018 17:32:57 -0600 Subject: [PATCH 160/308] Don't let the newsfragments wink out of existence --- changelogs/README.md | 2 +- changelogs/client_server/newsfragments/.gitignore | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelogs/client_server/newsfragments/.gitignore diff --git a/changelogs/README.md b/changelogs/README.md index 10d69cb6..a5fb1fb7 100644 --- a/changelogs/README.md +++ b/changelogs/README.md @@ -52,4 +52,4 @@ For this example, we're going to pretend that the `server_server` API doesn't ex name = "Spec Clarifications" showcontent = true ``` -4. Use the changelog in whatever RST documents you need +4. Create a `.gitignore` in `changelogs/server_server/newsfragments` with the contents `!.gitignore` diff --git a/changelogs/client_server/newsfragments/.gitignore b/changelogs/client_server/newsfragments/.gitignore new file mode 100644 index 00000000..b722e9e1 --- /dev/null +++ b/changelogs/client_server/newsfragments/.gitignore @@ -0,0 +1 @@ +!.gitignore \ No newline at end of file From a4f8e0807acee23f58b2a9e4ae2b0b0ba9f21cfa Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 9 Jul 2018 08:41:20 -0600 Subject: [PATCH 161/308] Print the python version being used in the CI build. --- scripts/test-and-build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/test-and-build.sh b/scripts/test-and-build.sh index e459a370..710b03dd 100755 --- a/scripts/test-and-build.sh +++ b/scripts/test-and-build.sh @@ -6,6 +6,11 @@ cd `dirname $0`/.. virtualenv -p python3 env . env/bin/activate + +# Print out the python versions for debugging purposes +python --version +pip --version + pip install -r scripts/requirements.txt # do sanity checks on the examples and swagger From 08f69128259bc3113614018e04b9342850388d2c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 9 Jul 2018 08:50:26 -0600 Subject: [PATCH 162/308] Ask Travis to install 3.5 explicitly --- .travis.yml | 2 +- scripts/test-and-build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 98dd14b8..013efdd9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ git: before_install: - sudo apt-get update -- sudo apt-get install python3 +- sudo apt-get install python3.5 # test-and-build does the installation, so tell travis to skip the # installation step diff --git a/scripts/test-and-build.sh b/scripts/test-and-build.sh index 710b03dd..397b620e 100755 --- a/scripts/test-and-build.sh +++ b/scripts/test-and-build.sh @@ -4,7 +4,7 @@ set -ex cd `dirname $0`/.. -virtualenv -p python3 env +virtualenv -p python3.5 env . env/bin/activate # Print out the python versions for debugging purposes From c65c6e2e70795dabd7717e195a72a39704e8a014 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 9 Jul 2018 12:25:34 -0600 Subject: [PATCH 163/308] Add changelog entries removed in the merge from master --- changelogs/client_server/newsfragments/1371.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1371.clarification diff --git a/changelogs/client_server/newsfragments/1371.clarification b/changelogs/client_server/newsfragments/1371.clarification new file mode 100644 index 00000000..88552fcd --- /dev/null +++ b/changelogs/client_server/newsfragments/1371.clarification @@ -0,0 +1 @@ +Mark ``GET /presence/{userId}/status`` as requiring authentication \ No newline at end of file From 7ada96020677b9df88422ac38b2b82754d974321 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 9 Jul 2018 12:28:27 -0600 Subject: [PATCH 164/308] Fix the Travis CI build to work on 3.5 --- .travis.yml | 9 +++------ scripts/requirements.txt | 1 + scripts/templating/matrix_templates/units.py | 7 ++++--- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 013efdd9..27e5cf20 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,13 +8,10 @@ sudo: false git: depth: 1 -before_install: +install: - sudo apt-get update -- sudo apt-get install python3.5 - -# test-and-build does the installation, so tell travis to skip the -# installation step -install: true +- sudo apt-get install python3.5 python3.5-dev +- sudo pip install --upgrade virtualenv script: - ./scripts/test-and-build.sh diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 6f7951d6..1f92c3dc 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -8,3 +8,4 @@ jsonschema >= 2.6.0 PyYAML >= 3.12 requests >= 2.18.4 towncrier == 18.6.0rc1 +six >= 1.11.0 \ No newline at end of file diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 0dbf8104..acd85d66 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -29,9 +29,10 @@ import os.path import re import subprocess import sys -import urllib.request, urllib.parse, urllib.error import yaml +import requests.utils from functools import reduce +from six.moves.urllib.parse import urlencode matrix_doc_dir=reduce(lambda acc,_: os.path.dirname(acc), list(range(1, 5)), os.path.abspath(__file__)) @@ -524,7 +525,7 @@ class MatrixUnits(Units): if param_loc == "path": path_template = path_template.replace( - "{%s}" % param_name, urllib.parse.quote(example) + "{%s}" % param_name, requests.utils.requote_uri(example) ) elif param_loc == "query": if type(example) == list: @@ -564,7 +565,7 @@ class MatrixUnits(Units): ) endpoint["res_headers"] = headers query_string = "" if len( - example_query_params) == 0 else "?" + urllib.parse.urlencode( + example_query_params) == 0 else "?" + urlencode( example_query_params) if example_body: endpoint["example"][ From 550f95570b327e8f57e832866427049603f48b06 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 9 Jul 2018 15:17:28 -0600 Subject: [PATCH 165/308] Fix regex for some versions of python --- scripts/gendoc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 31ee9e69..25fd4577 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -283,12 +283,12 @@ def addAnchors(path): with open(path, "rb") as f: lines = f.readlines() - replacement = replacement = r'

\n\1' + replacement = r'

\n\1' with open(path, "wb") as f: for line in lines: line = line.decode("UTF-8") line = re.sub(r'()', replacement, line.rstrip()) - line = re.sub(r'(
)', replacement, line.rstrip()) + line = re.sub(r'(
)', replacement, line.rstrip()) f.write((line + "\n").encode('UTF-8')) From 423d5593f5a48e8bd0b7dfea0d37278b957d7a56 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 9 Jul 2018 15:40:03 -0600 Subject: [PATCH 166/308] Generify how OPTIONS and CORS are handled --- specification/client_server_api.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 33ee8b1a..8dde4302 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -173,10 +173,8 @@ to pre-flight requests and supply Cross-Origin Resource Sharing (CORS) headers o all requests. When a client approaches the server with a pre-flight (``OPTIONS``) request, the -server should respond with the CORS headers for that route. If the route does not -exist, the server should return an ``M_NOT_FOUND`` error with a 404 status code. - -The standard CORS headers to be returned by servers on all requests are: +server should respond with the CORS headers for that route. The recommended CORS +headers to be returned by servers on all requests are: .. code:: From 3a9cfd490feaeb6a91d14ade329393b50cb1337b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 9 Jul 2018 23:52:22 -0600 Subject: [PATCH 167/308] Improve pagination handling in proposals.py Previously if multiple pages were returned then the script would lose the results. A check to ensure the response is an array was also added. This helps avoid somewhat cryptic errors when iterating over non-lists (eg: error messages, bad output, etc) --- scripts/proposals.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index b489f0d2..eec6d652 100755 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -12,13 +12,18 @@ authors = set() prs = set() def getpage(url, page): - resp = requests.get(url + str(page)) + url = url + str(page) + resp = requests.get(url) for link in resp.links.values(): if link['rel'] == 'last': pagecount = re.search('page=(.+?)', link['url']).group(1) - return resp.json() + val = resp.json() + if not isinstance(val, list): + print(val) # Just dump the raw (likely error) response to the log + raise Exception("Error calling %s" % url) + return val def getbylabel(label): pagecount = 1 @@ -27,7 +32,7 @@ def getbylabel(label): print(urlbase) json.extend(getpage(urlbase, 1)) for page in range(2, int(pagecount) + 1): - getpage(urlbase, page) + json.extend(getpage(urlbase, page)) return json From e9f7d2096e100c27c530ec4660670bdb9f8605a4 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 08:39:34 -0600 Subject: [PATCH 168/308] Add changelog entry for CORS --- changelogs/client_server/newsfragments/1365.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1365.feature diff --git a/changelogs/client_server/newsfragments/1365.feature b/changelogs/client_server/newsfragments/1365.feature new file mode 100644 index 00000000..d2864e96 --- /dev/null +++ b/changelogs/client_server/newsfragments/1365.feature @@ -0,0 +1 @@ +Document the CORS/preflight headers \ No newline at end of file From 55e4e90ab1155c5b4d4031fe5108c7b5ea63ac53 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 12:52:11 -0600 Subject: [PATCH 169/308] Document message formats as-is This is likely to later be replaced by mixins, however this is being documented now so clients aren't left in the dark. Fixes https://github.com/matrix-org/matrix-doc/issues/917 --- event-schemas/examples/m.room.message#m.emote | 4 +++- event-schemas/examples/m.room.message#m.text | 4 +++- event-schemas/schema/m.room.message#m.emote | 10 ++++++++++ event-schemas/schema/m.room.message#m.text | 10 ++++++++++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/event-schemas/examples/m.room.message#m.emote b/event-schemas/examples/m.room.message#m.emote index 4280928e..79292ddf 100644 --- a/event-schemas/examples/m.room.message#m.emote +++ b/event-schemas/examples/m.room.message#m.emote @@ -2,7 +2,9 @@ "age": 242352, "content": { "body": "thinks this is an example emote", - "msgtype": "m.emote" + "msgtype": "m.emote", + "format": "org.matrix.custom.html", + "formatted_body": "thinks this is an example emote" }, "origin_server_ts": 1431961217939, "event_id": "$WLGTSEFSEF:localhost", diff --git a/event-schemas/examples/m.room.message#m.text b/event-schemas/examples/m.room.message#m.text index e00c7aa5..48a97db8 100644 --- a/event-schemas/examples/m.room.message#m.text +++ b/event-schemas/examples/m.room.message#m.text @@ -2,7 +2,9 @@ "age": 242352, "content": { "body": "This is an example text message", - "msgtype": "m.text" + "msgtype": "m.text", + "format": "org.matrix.custom.html", + "formatted_body": "This is an example text message" }, "origin_server_ts": 1431961217939, "event_id": "$WLGTSEFSEF:localhost", diff --git a/event-schemas/schema/m.room.message#m.emote b/event-schemas/schema/m.room.message#m.emote index 88860cb2..f67a184b 100644 --- a/event-schemas/schema/m.room.message#m.emote +++ b/event-schemas/schema/m.room.message#m.emote @@ -12,6 +12,16 @@ properties: enum: - m.emote type: string + format: + description: |- + The format used in the ``formatted_body``. Currently only + ``org.matrix.custom.html`` is supported. + type: string + formatted_body: + description: |- + The formatted version of the ``body``. This is required if ``format`` + is specified. + type: string required: - msgtype - body diff --git a/event-schemas/schema/m.room.message#m.text b/event-schemas/schema/m.room.message#m.text index 2720172d..b481bcea 100644 --- a/event-schemas/schema/m.room.message#m.text +++ b/event-schemas/schema/m.room.message#m.text @@ -12,6 +12,16 @@ properties: enum: - m.text type: string + format: + description: |- + The format used in the ``formatted_body``. Currently only + ``org.matrix.custom.html`` is supported. + type: string + formatted_body: + description: |- + The formatted version of the ``body``. This is required if ``format`` + is specified. + type: string required: - msgtype - body From c999b7c2e21b61a8f5d217054555d6a317a76cf0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 13:00:37 -0600 Subject: [PATCH 170/308] Add newsfragment --- changelogs/client_server/newsfragments/1397.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1397.feature diff --git a/changelogs/client_server/newsfragments/1397.feature b/changelogs/client_server/newsfragments/1397.feature new file mode 100644 index 00000000..e4bd248a --- /dev/null +++ b/changelogs/client_server/newsfragments/1397.feature @@ -0,0 +1 @@ +Document message formats on ``m.text`` and ``m.emote`` messages From fbee83d5cf53ddb37ce4b54b6f12d7cbe0496038 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 15:08:28 -0600 Subject: [PATCH 171/308] Clarify some of the properties on the search result Fixes https://github.com/matrix-org/matrix-doc/issues/773 --- api/client-server/search.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/api/client-server/search.yaml b/api/client-server/search.yaml index 687140b3..e28461c4 100644 --- a/api/client-server/search.yaml +++ b/api/client-server/search.yaml @@ -227,6 +227,9 @@ paths: description: |- The historic profile information of the users that sent the events returned. + + The ``string`` key is the user ID for which + the profile belongs to. additionalProperties: type: object title: User Profile @@ -260,15 +263,24 @@ paths: The current state for every room in the results. This is included if the request had the ``include_state`` key set with a value of ``true``. + + The ``string`` key is the room ID for which the ``State + Event`` array belongs to. additionalProperties: type: array title: Room State items: + type: object "$ref": "definitions/event-schemas/schema/core-event-schema/state_event.yaml" groups: type: object title: Groups - description: Any groups that were requested. + description: |- + Any groups that were requested. + + The outer ``string`` key is the group key requested (eg: ``room_id`` + or ``sender``). The inner ``string`` key is the grouped value (eg: + a room's ID or a user's ID). additionalProperties: type: object title: Group Key From b6f9e13d469718aa07c879b88136f4ec99719505 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 15:09:53 -0600 Subject: [PATCH 172/308] Add newsfragment --- changelogs/client_server/newsfragments/1400.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1400.clarification diff --git a/changelogs/client_server/newsfragments/1400.clarification b/changelogs/client_server/newsfragments/1400.clarification new file mode 100644 index 00000000..3fd29e92 --- /dev/null +++ b/changelogs/client_server/newsfragments/1400.clarification @@ -0,0 +1 @@ +Clarify some of the properties on the search result From 9e0fafbcd4436d3d245a942117eccb1d2396261d Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 15:13:22 -0600 Subject: [PATCH 173/308] Remove debugging line --- scripts/gendoc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/gendoc.py b/scripts/gendoc.py index 25fd4577..af613b68 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -405,8 +405,7 @@ def logv(line): def cleanup_env(): - #shutil.rmtree(tmp_dir) - pass + shutil.rmtree(tmp_dir) def mkdirp(d) : From ebc7db12fbb9e5ff0f6193605da8182710d0937c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 16:52:17 -0600 Subject: [PATCH 174/308] Remove extraneous list casting --- api/check_examples.py | 6 +++--- scripts/dump-swagger.py | 6 +++--- scripts/gendoc.py | 16 ++++++++-------- scripts/proposals.py | 2 +- scripts/templating/batesian/__init__.py | 2 +- scripts/templating/batesian/sections.py | 2 +- scripts/templating/build.py | 4 ++-- scripts/templating/matrix_templates/sections.py | 4 ++-- scripts/templating/matrix_templates/units.py | 12 ++++++------ 9 files changed, 27 insertions(+), 27 deletions(-) diff --git a/api/check_examples.py b/api/check_examples.py index 3ecd5a4c..be0676bb 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -88,9 +88,9 @@ def check_swagger_file(filepath): with open(filepath) as f: swagger = yaml.load(f) - for path, path_api in list(swagger.get('paths', {}).items()): + for path, path_api in swagger.get('paths', {}).items(): - for method, request_api in list(path_api.items()): + for method, request_api in path_api.items(): request = "%s %s" % (method.upper(), path) for parameter in request_api.get('parameters', ()): if parameter['in'] == 'body': @@ -100,7 +100,7 @@ def check_swagger_file(filepath): responses = request_api['responses'] except KeyError: raise ValueError("No responses for %r" % (request,)) - for code, response in list(responses.items()): + for code, response in responses.items(): check_response(filepath, request, code, response) diff --git a/scripts/dump-swagger.py b/scripts/dump-swagger.py index 6dd9864f..7994324f 100755 --- a/scripts/dump-swagger.py +++ b/scripts/dump-swagger.py @@ -94,11 +94,11 @@ for filename in os.listdir(cs_api_dir): api = units.resolve_references(filepath, api) basePath = api['basePath'] - for path, methods in list(api["paths"].items()): + for path, methods in api["paths"].items(): path = (basePath + path).replace('%CLIENT_MAJOR_VERSION%', major_version) - for method, spec in list(methods.items()): - if "tags" in list(spec.keys()): + for method, spec in methods.items(): + if "tags" in spec.keys(): if path not in output["paths"]: output["paths"][path] = {} output["paths"][path][method] = spec diff --git a/scripts/gendoc.py b/scripts/gendoc.py index af613b68..16c40af5 100755 --- a/scripts/gendoc.py +++ b/scripts/gendoc.py @@ -299,7 +299,7 @@ def run_through_template(input_files, set_verbose, substitutions): "-i", "matrix_templates", ] - for k, v in list(substitutions.items()): + for k, v in substitutions.items(): args.append("--substitution=%s=%s" % (k, v)) if set_verbose: @@ -359,7 +359,7 @@ def get_build_target(all_targets, target_name): for i, entry in enumerate(group): if isinstance(entry, dict): group[i] = { - (rel_depth + depth): v for (rel_depth, v) in list(entry.items()) + (rel_depth + depth): v for (rel_depth, v) in entry.items() } return group @@ -378,7 +378,7 @@ def get_build_target(all_targets, target_name): # file_entry is a dict which has more file entries as values elif isinstance(file_entry, dict): resolved_entry = {} - for (depth, entry) in list(file_entry.items()): + for (depth, entry) in file_entry.items(): if not isinstance(entry, str): raise Exception( "Double-nested depths are not supported. Entry: %s" % (file_entry,) @@ -429,7 +429,7 @@ def main(targets, dest_dir, keep_intermediates, substitutions): target_defs = yaml.load(targ_file.read()) if targets == ["all"]: - targets = list(target_defs["targets"].keys()) + targets = target_defs["targets"].keys() log("Building spec [targets=%s]" % targets) @@ -443,17 +443,17 @@ def main(targets, dest_dir, keep_intermediates, substitutions): templated_files[target_name] = templated_file # we do all the templating at once, because it's slow - run_through_template(list(templated_files.values()), VERBOSE, substitutions) + run_through_template(templated_files.values(), VERBOSE, substitutions) stylesheets = glob.glob(os.path.join(script_dir, "css", "*.css")) - for target_name, templated_file in list(templated_files.items()): + for target_name, templated_file in templated_files.items(): target = target_defs["targets"].get(target_name) version_label = None if target: version_label = target.get("version_label") if version_label: - for old, new in list(substitutions.items()): + for old, new in substitutions.items(): version_label = version_label.replace(old, new) rst_file = os.path.join(tmp_dir, "spec_%s.rst" % (target_name,)) @@ -481,7 +481,7 @@ def main(targets, dest_dir, keep_intermediates, substitutions): def list_targets(): with open(os.path.join(spec_dir, "targets.yaml"), "r") as targ_file: target_defs = yaml.load(targ_file.read()) - targets = list(target_defs["targets"].keys()) + targets = target_defs["targets"].keys() print("\n".join(targets)) diff --git a/scripts/proposals.py b/scripts/proposals.py index 156d5351..b489f0d2 100755 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -14,7 +14,7 @@ prs = set() def getpage(url, page): resp = requests.get(url + str(page)) - for link in list(resp.links.values()): + for link in resp.links.values(): if link['rel'] == 'last': pagecount = re.search('page=(.+?)', link['url']).group(1) diff --git a/scripts/templating/batesian/__init__.py b/scripts/templating/batesian/__init__.py index f5d83a1e..e901590f 100644 --- a/scripts/templating/batesian/__init__.py +++ b/scripts/templating/batesian/__init__.py @@ -24,7 +24,7 @@ class AccessKeyStore(object): self.accessed_set = set() def keys(self): - return list(self.data.keys()) + return self.data.keys() def add(self, key, unit_dict): self.data[key] = unit_dict diff --git a/scripts/templating/batesian/sections.py b/scripts/templating/batesian/sections.py index 2e5593d6..18a622f6 100644 --- a/scripts/templating/batesian/sections.py +++ b/scripts/templating/batesian/sections.py @@ -54,7 +54,7 @@ class Sections(object): ) elif isinstance(section, dict): self.log(" Generated multiple sections:") - for (k, v) in list(section.items()): + for (k, v) in section.items(): if not isinstance(k, str) or not isinstance(v, str): raise Exception( ("Method %s returned multiple sections as a dict but " + diff --git a/scripts/templating/build.py b/scripts/templating/build.py index de9ead35..fae4db56 100755 --- a/scripts/templating/build.py +++ b/scripts/templating/build.py @@ -169,7 +169,7 @@ def main(input_module, files=None, out_dir=None, verbose=False, substitutions={} # print out valid section keys if no file supplied if not files: print("\nValid template variables:") - for key in list(sections.keys()): + for key in sections.keys(): sec_text = "" if (len(sections[key]) > 75) else ( "(Value: '%s')" % sections[key] ) @@ -211,7 +211,7 @@ def process_file(env, sections, filename, output_filename): # Do these substitutions outside of the ordinary templating system because # we want them to apply to things like the underlying swagger used to # generate the templates, not just the top-level sections. - for old, new in list(substitutions.items()): + for old, new in substitutions.items(): output = output.replace(old, new) with open(output_filename, "wb") as f: diff --git a/scripts/templating/matrix_templates/sections.py b/scripts/templating/matrix_templates/sections.py index 71c1acf3..64e32aa4 100644 --- a/scripts/templating/matrix_templates/sections.py +++ b/scripts/templating/matrix_templates/sections.py @@ -86,7 +86,7 @@ class MatrixSections(Sections): # the key is the section name and the value is the value of the section def render_group_http_apis(self): # map all swagger_apis to the form $GROUP_http_api - swagger_groups = list(self.units.get("swagger_apis").keys()) + swagger_groups = self.units.get("swagger_apis").keys() renders = {} for group in swagger_groups: sortFnOrPathList = None @@ -134,7 +134,7 @@ class MatrixSections(Sections): "m.room.message#m.file" ] other_msgtypes = [ - k for k in list(schemas.keys()) if k.startswith("m.room.message#") and + k for k in schemas.keys() if k.startswith("m.room.message#") and k not in msgtype_order ] for event_name in (msgtype_order + other_msgtypes): diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index acd85d66..63a4e5a8 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -35,7 +35,7 @@ from functools import reduce from six.moves.urllib.parse import urlencode matrix_doc_dir=reduce(lambda acc,_: os.path.dirname(acc), - list(range(1, 5)), os.path.abspath(__file__)) + range(1, 5), os.path.abspath(__file__)) HTTP_APIS = { os.path.join(matrix_doc_dir, "api/application-service"): "as", @@ -126,7 +126,7 @@ def resolve_references(path, schema): else: result = OrderedDict() - for key, value in list(schema.items()): + for key, value in schema.items(): result[key] = resolve_references(path, value) return result elif isinstance(schema, list): @@ -211,7 +211,7 @@ def get_json_schema_object_fields(obj, enforce_title=False): props = obj.get("patternProperties") if props: # try to replace horrible regex key names with pretty x-pattern ones - for key_name in list(props.keys()): + for key_name in props.keys(): pretty_key = props[key_name].get("x-pattern") if pretty_key: props[pretty_key] = props[key_name] @@ -382,7 +382,7 @@ def get_example_for_schema(schema): if 'properties' not in schema: raise Exception('"object" property has neither properties nor example') res = OrderedDict() - for prop_name, prop in list(schema['properties'].items()): + for prop_name, prop in schema['properties'].items(): logger.debug("Parsing property %r" % prop_name) prop_example = get_example_for_schema(prop) res[prop_name] = prop_example @@ -558,7 +558,7 @@ class MatrixUnits(Units): ) if "headers" in good_response: headers = TypeTable() - for (header_name, header) in list(good_response["headers"].items()): + for (header_name, header) in good_response["headers"].items(): headers.add_row( TypeTableRow(key=header_name, title=header["type"], desc=header["description"]), @@ -617,7 +617,7 @@ class MatrixUnits(Units): def load_swagger_apis(self): apis = {} - for path, suffix in list(HTTP_APIS.items()): + for path, suffix in HTTP_APIS.items(): for filename in os.listdir(path): if not filename.endswith(".yaml"): continue From 905ef6daddbaeff457c9f265d0d2bf64332c558b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 10 Jul 2018 16:57:21 -0600 Subject: [PATCH 175/308] Have the towncrier generator complain if the output looks wrong --- scripts/templating/matrix_templates/units.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 63a4e5a8..9b3d621d 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -845,7 +845,14 @@ class MatrixUnits(Units): stderr=subprocess.PIPE, cwd=tc_path, ).strip().decode('UTF-8') - except subprocess.CalledProcessError: + + # This is a bit of a hack, but it does mean that the log at least gets *something* + # to tell us it broke + if not raw_log.startswith("Unreleased Changes"): + logger.error(raw_log) + raw_log = "" + except subprocess.CalledProcessError as e: + logger.error(e) raw_log = "" tc_lines = raw_log.splitlines() From a8461e647f2c5c6bf8cdb4f5157736ed2589948a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 11 Jul 2018 08:35:55 -0600 Subject: [PATCH 176/308] Improve the error handling for towncrier The changelog shows up via stdout, everything else via stderr. We dump as much information as we can into the changelog to make debugging errors easier. --- scripts/templating/matrix_templates/units.py | 25 +++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 9b3d621d..760aeb6b 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -839,21 +839,28 @@ class MatrixUnits(Units): tc_lines = [] if os.path.isdir(tc_path): logger.info("Generating towncrier changelog for: %s" % name) - try: - raw_log = subprocess.check_output( - ['towncrier', '--version', 'Unreleased Changes', '--name', name, '--draft'], - stderr=subprocess.PIPE, - cwd=tc_path, - ).strip().decode('UTF-8') + p = subprocess.run( + ['towncrier', '--version', 'Unreleased Changes', '--name', name, '--draft'], + cwd=tc_path, + check=False, # We'll manually check the exit code + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + if p.returncode != 0: + # Something broke - dump as much information as we can + logger.error("Towncrier exited with code %s" % p.returncode) + logger.error(p.stdout.decode('UTF-8')) + logger.error(p.stderr.decode('UTF-8')) + raw_log = "" + else: + raw_log = p.stdout.decode('UTF-8') # This is a bit of a hack, but it does mean that the log at least gets *something* # to tell us it broke if not raw_log.startswith("Unreleased Changes"): + logger.error("Towncrier appears to have failed to generate a changelog") logger.error(raw_log) raw_log = "" - except subprocess.CalledProcessError as e: - logger.error(e) - raw_log = "" tc_lines = raw_log.splitlines() title_part = None From bd2c0b7c98612635c514567094caccda974fafba Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 11 Jul 2018 13:09:19 -0600 Subject: [PATCH 177/308] Convert server keys to swagger --- api/server-server/definitions/keys.yaml | 96 ++++++++++++++ .../definitions/keys_query_response.yaml | 27 ++++ api/server-server/examples/server_key.json | 23 ++++ api/server-server/keys.yaml | 121 ++++++++++++++++++ specification/server_server_api.rst | 34 +---- 5 files changed, 269 insertions(+), 32 deletions(-) create mode 100644 api/server-server/definitions/keys.yaml create mode 100644 api/server-server/definitions/keys_query_response.yaml create mode 100644 api/server-server/examples/server_key.json create mode 100644 api/server-server/keys.yaml diff --git a/api/server-server/definitions/keys.yaml b/api/server-server/definitions/keys.yaml new file mode 100644 index 00000000..5d1c39de --- /dev/null +++ b/api/server-server/definitions/keys.yaml @@ -0,0 +1,96 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +title: Server Keys +description: Server keys +example: + $ref: "../examples/server_key.json" +properties: + server_name: + type: string + description: DNS name of the homeserver + required: true # TODO: Verify + example: "example.org" + verify_keys: + type: object + description: Public keys of the homeserver for verifying digital signatures + required: true # TODO: Verify + additionalProperties: + type: object + title: Verify Key + example: { + "ed25519:auto2": { + "key": "Base+64+Encoded+Signature+Verification+Key" + } + } + properties: + key: + type: string + description: The key + required: true + example: "Base+64+Encoded+Signature+Verification+Key" + old_verify_keys: + type: object + description: The public keys that the server used to use and when it stopped using them + additionalProperties: + type: object + title: Old Verify Key + example: { + "ed25519:auto1": { + "expired_ts": 922834800000, + "key": "Base+64+Encoded+Signature+Verification+Key" + } + } + properties: + expired_ts: + type: integer + format: int64 + description: The expiration time + required: true + example: 922834800000 + key: + type: string + description: The key + required: true + example: "Base+64+Encoded+Signature+Verification+Key" + signatures: + type: object + description: Digital signatures for this object signed using the ``verify_keys`` + additionalProperties: + type: object + title: Signed Server + example: { + "example.org": { + "ad25519:auto2": "Base+64+Encoded+Signature+Verification+Key" + } + } + additionalProperties: + type: string + name: Encoded Signature Verification Key + tls_fingerprints: + type: array + description: Hashes of X.509 TLS certificates used by this server encoded as `Unpadded Base64`_ + items: + type: object + title: TLS Fingerprint + properties: + sha256: + type: string + description: The encoded fingerprint + example: Base+64+Encoded+SHA-256-Fingerprint + valid_until_ts: + type: integer + format: int64 + description: POSIX timestamp when the list of valid keys should be refreshed + example: 1052262000000 diff --git a/api/server-server/definitions/keys_query_response.yaml b/api/server-server/definitions/keys_query_response.yaml new file mode 100644 index 00000000..8223f9ec --- /dev/null +++ b/api/server-server/definitions/keys_query_response.yaml @@ -0,0 +1,27 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +description: Server keys +example: { + "server_keys": [{ + $ref: "../examples/server_key.json" + }] +} +properties: + server_keys: + type: array + title: Server Keys + description: The server keys + items: + $ref: "keys.yaml" diff --git a/api/server-server/examples/server_key.json b/api/server-server/examples/server_key.json new file mode 100644 index 00000000..a3934bd3 --- /dev/null +++ b/api/server-server/examples/server_key.json @@ -0,0 +1,23 @@ +{ + "server_name": "example.org", + "verify_keys": { + "ed25519:auto2": { + "key": "Base+64+Encoded+Signature+Verification+Key" + } + }, + "old_verify_keys": { + "ed25519:auto1": { + "expired_ts": 922834800000, + "key": "Base+64+Encoded+Old+Verify+Key" + } + }, + "signatures": { + "example.org": { + "ed25519:auto2": "Base+64+Encoded+Signature" + } + }, + "tls_fingerprints": [{ + "sha256": "Base+64+Encoded+SHA-256-Fingerprint" + }], + "valid_until_ts": 1052262000000 +} \ No newline at end of file diff --git a/api/server-server/keys.yaml b/api/server-server/keys.yaml new file mode 100644 index 00000000..e8e5167d --- /dev/null +++ b/api/server-server/keys.yaml @@ -0,0 +1,121 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Key Exchange API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/key/v2 +produces: + - application/json +paths: + "/server/{keyId}": + get: + summary: Get the server's key + description: Get the server's key + operationId: getServerKey + parameters: + - in: path + name: keyId + type: string + description: Key ID + required: false + x-example: TODO # No examples in the spec so far + responses: + 200: + description: The server's keys + schema: + $ref: "definitions/keys.yaml" + "/query/{serverName}/{keyId}": + get: + summary: Retreive a server key + description: Retreive a server key + operationId: getQueryKeys + parameters: + - in: path + name: serverName + type: string + description: Server name + required: true + x-example: matrix.org + - in: path + name: keyId + type: string + description: Key ID + required: true + x-example: TODO # No examples in spec so far + - in: query + name: minimum_valid_until_ts + type: integer + format: int64 + description: Minimum Valid Until Milliseconds + required: true # TODO: Verify + x-example: 1234567890 + responses: + 200: + description: The keys for the server + schema: + $ref: "definitions/keys_query_response.yaml" + "/query": + post: + summary: Retreive a server key + description: Retreive a server key + operationId: postQueryKeys + parameters: + - in: body + name: body + schema: + type: object + # TODO: Improve example + example: { + "server_keys": { + "{server_name}": { + "{key_id}": { + "minimum_valid_until_ts": 1234567890 + } + } + } + } + properties: + server_keys: + type: object + description: The query criteria + required: true + additionalProperties: + type: object + name: ServerName + description: The server names to query + additionalProperties: + type: array + name: ServerKey + description: The server keys to query + items: + type: object + title: Query Criteria + description: The query criteria + properties: + minimum_valid_until_ts: + type: integer + format: int64 + description: Minimum Valid Until MS + required: true # TODO: Verify + x-example: 1234567890 + responses: + 200: + description: The keys for the server + schema: + $ref: "definitions/keys_query_response.yaml" diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f2b3204a..86151991 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -1,5 +1,6 @@ .. Copyright 2016 OpenMarket Ltd .. Copyright 2017 New Vector Ltd +.. Copyright 2018 New Vector Ltd .. .. Licensed under the Apache License, Version 2.0 (the "License"); .. you may not use this file except in compliance with the License. @@ -232,38 +233,7 @@ This API can return keys for servers that are offline by using cached responses taken from when the server was online. Keys can be queried from multiple servers to mitigate against DNS spoofing. -Example request: - -.. code:: - - GET /_matrix/key/v2/query/{server_name}/{key_id}/?minimum_valid_until_ts={minimum_valid_until_ts} HTTP/1.1 - - POST /_matrix/key/v2/query HTTP/1.1 - Content-Type: application/json - - { - "server_keys": { - "{server_name}": { - "{key_id}": { - "minimum_valid_until_ts": {posix_timestamp} - } - } - } - } - - -Response: - -.. code:: - - HTTP/1.1 200 OK - Content-Type: application/json - { - "server_keys": [ - # List of responses with same format as /_matrix/key/v2/server - # signed by both the originating server and this server. - ] - } +{{keys_ss_http_api}} Version 1 +++++++++ From 96889f16e93aed1a273d0e9ef4621ab079b64f32 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 11 Jul 2018 13:20:44 -0600 Subject: [PATCH 178/308] Split out and fix the /server and /query key APIs --- .../{keys.yaml => keys_query.yaml} | 39 ++++------------ api/server-server/keys_server.yaml | 42 +++++++++++++++++ specification/server_server_api.rst | 46 +------------------ 3 files changed, 53 insertions(+), 74 deletions(-) rename api/server-server/{keys.yaml => keys_query.yaml} (72%) create mode 100644 api/server-server/keys_server.yaml diff --git a/api/server-server/keys.yaml b/api/server-server/keys_query.yaml similarity index 72% rename from api/server-server/keys.yaml rename to api/server-server/keys_query.yaml index e8e5167d..c0811dbc 100644 --- a/api/server-server/keys.yaml +++ b/api/server-server/keys_query.yaml @@ -23,23 +23,6 @@ basePath: /_matrix/key/v2 produces: - application/json paths: - "/server/{keyId}": - get: - summary: Get the server's key - description: Get the server's key - operationId: getServerKey - parameters: - - in: path - name: keyId - type: string - description: Key ID - required: false - x-example: TODO # No examples in the spec so far - responses: - 200: - description: The server's keys - schema: - $ref: "definitions/keys.yaml" "/query/{serverName}/{keyId}": get: summary: Retreive a server key @@ -100,20 +83,16 @@ paths: name: ServerName description: The server names to query additionalProperties: - type: array - name: ServerKey + type: object + title: Query Criteria description: The server keys to query - items: - type: object - title: Query Criteria - description: The query criteria - properties: - minimum_valid_until_ts: - type: integer - format: int64 - description: Minimum Valid Until MS - required: true # TODO: Verify - x-example: 1234567890 + properties: + minimum_valid_until_ts: + type: integer + format: int64 + description: Minimum Valid Until MS + required: true # TODO: Verify + example: 1234567890 responses: 200: description: The keys for the server diff --git a/api/server-server/keys_server.yaml b/api/server-server/keys_server.yaml new file mode 100644 index 00000000..819a2599 --- /dev/null +++ b/api/server-server/keys_server.yaml @@ -0,0 +1,42 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Key Exchange API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/key/v2 +produces: + - application/json +paths: + "/server/{keyId}": + get: + summary: Get the server's key + description: Get the server's key + operationId: getServerKey + parameters: + - in: path + name: keyId + type: string + description: Key ID + required: false + x-example: TODO # No examples in the spec so far + responses: + 200: + description: The server's keys + schema: + $ref: "definitions/keys.yaml" diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 86151991..92507687 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -167,51 +167,9 @@ If a server goes offline intermediate notary servers should continue to return the last response they received from that server so that the signatures of old events sent by that server can still be checked. -==================== =================== ====================================== - Key Type Description -==================== =================== ====================================== -``server_name`` String DNS name of the homeserver. -``verify_keys`` Object Public keys of the homeserver for - verifying digital signatures. -``old_verify_keys`` Object The public keys that the server used - to use and when it stopped using them. -``signatures`` Object Digital signatures for this object - signed using the ``verify_keys``. -``tls_fingerprints`` Array of Objects Hashes of X.509 TLS certificates used - by this server encoded as `Unpadded Base64`_. -``valid_until_ts`` Integer POSIX timestamp when the list of valid - keys should be refreshed. -==================== =================== ====================================== +{{keys_server_ss_http_api}} -.. code:: json - - { - "old_verify_keys": { - "ed25519:auto1": { - "expired_ts": 922834800000, - "key": "Base+64+Encoded+Old+Verify+Key" - } - }, - "server_name": "example.org", - "signatures": { - "example.org": { - "ed25519:auto2": "Base+64+Encoded+Signature" - } - }, - "tls_fingerprints": [ - { - "sha256": "Base+64+Encoded+SHA-256-Fingerprint" - } - ], - "valid_until_ts": 1052262000000, - "verify_keys": { - "ed25519:auto2": { - "key": "Base+64+Encoded+Signature+Verification+Key" - } - } - } - Querying Keys Through Another Server ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -233,7 +191,7 @@ This API can return keys for servers that are offline by using cached responses taken from when the server was online. Keys can be queried from multiple servers to mitigate against DNS spoofing. -{{keys_ss_http_api}} +{{keys_query_ss_http_api}} Version 1 +++++++++ From 3e13ec2841e2d8c0a67a4e4ae595734d5dc3e757 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 11 Jul 2018 13:43:15 -0600 Subject: [PATCH 179/308] Fix required properties in POST /query --- api/server-server/keys_query.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/server-server/keys_query.yaml b/api/server-server/keys_query.yaml index c0811dbc..1ba0f5ba 100644 --- a/api/server-server/keys_query.yaml +++ b/api/server-server/keys_query.yaml @@ -77,7 +77,6 @@ paths: server_keys: type: object description: The query criteria - required: true additionalProperties: type: object name: ServerName @@ -91,8 +90,8 @@ paths: type: integer format: int64 description: Minimum Valid Until MS - required: true # TODO: Verify example: 1234567890 + required: ['server_keys'] responses: 200: description: The keys for the server From 374ec00046d0366c88a8e6cfd40c70c9624ed264 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 11 Jul 2018 16:51:24 -0600 Subject: [PATCH 180/308] Convert things that mention "Transaction" to swagger There's two kinds of transactions currently: one with EDUs and one without. The one with EDUs is only used on /send, however the schema is still somewhat worth splitting out for simplicity. The examples are brought apart to make them slightly more reusable for when they get dumped into the relevant sections of the spec (see TODO in server_server_api.rst) Further, the Transactions stuff introduces tuples to the spec. The units.py has been updated to support this. --- api/server-server/definitions/edu.yaml | 37 +++++ .../definitions/full_transaction.yaml | 28 ++++ api/server-server/definitions/pdu.yaml | 146 ++++++++++++++++++ .../definitions/transaction.yaml | 35 +++++ api/server-server/events.yaml | 118 ++++++++++++++ api/server-server/examples/edu.json | 8 + .../examples/full_transaction.json | 6 + api/server-server/examples/pdu.json | 25 +++ api/server-server/examples/transaction.json | 5 + api/server-server/transactions.yaml | 51 ++++++ scripts/templating/matrix_templates/units.py | 25 ++- specification/server_server_api.rst | 55 +------ 12 files changed, 485 insertions(+), 54 deletions(-) create mode 100644 api/server-server/definitions/edu.yaml create mode 100644 api/server-server/definitions/full_transaction.yaml create mode 100644 api/server-server/definitions/pdu.yaml create mode 100644 api/server-server/definitions/transaction.yaml create mode 100644 api/server-server/events.yaml create mode 100644 api/server-server/examples/edu.json create mode 100644 api/server-server/examples/full_transaction.json create mode 100644 api/server-server/examples/pdu.json create mode 100644 api/server-server/examples/transaction.json create mode 100644 api/server-server/transactions.yaml diff --git a/api/server-server/definitions/edu.yaml b/api/server-server/definitions/edu.yaml new file mode 100644 index 00000000..4597ca62 --- /dev/null +++ b/api/server-server/definitions/edu.yaml @@ -0,0 +1,37 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: Address any concerns about this being inaccurate (flagged as such in the EDUs section) + +type: object +title: Ephemeral Data Unit +description: An ephemeral data unit +example: + $ref: "../examples/edu.json" +properties: + edu_type: + type: string + description: The type of ephemeral message + example: "!abc123:matrix.org" + origin: + type: string + description: The server name sending the ephemeral message + example: "matrix.org" + destination: + type: string + description: The server name receiving the ephemeral message + example: "elsewhere.com" + content: + type: object + description: The content of the ephemeral message diff --git a/api/server-server/definitions/full_transaction.yaml b/api/server-server/definitions/full_transaction.yaml new file mode 100644 index 00000000..39250433 --- /dev/null +++ b/api/server-server/definitions/full_transaction.yaml @@ -0,0 +1,28 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +title: Transaction +description: Transaction +example: + $ref: "../examples/full_transaction.json" +allOf: + - $ref: "transaction.yaml" + - type: object + properties: + edus: + type: array + description: List of ephemeral messages. May be omitted if there are no ephemeral messages to be sent. + items: + $ref: "edu.yaml" +required: ['origin', 'origin_server_ts', 'pdus'] \ No newline at end of file diff --git a/api/server-server/definitions/pdu.yaml b/api/server-server/definitions/pdu.yaml new file mode 100644 index 00000000..b3f25ccf --- /dev/null +++ b/api/server-server/definitions/pdu.yaml @@ -0,0 +1,146 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +title: Persistent Data Unit +description: A persistent data unit (event) +example: + $ref: "../examples/pdu.json" +properties: + room_id: + type: string + description: Room identifier + example: "!abc123:matrix.org" + sender: + type: string + description: The ID of the user sending the event + example: "@someone:matrix.org" + origin: + type: string + description: The ``server_name`` of the homeserver that created this event + example: "matrix.org" + origin_server_ts: + type: integer + format: int64 + description: Timestamp in milliseconds on origin homeserver when this event was created. + example: 1234567890 + type: + type: string + description: Event type + required: true + example: "m.room.message" + state_key: + type: string + description: |- + If this key is present, the event is a state event, and it will replace previous events + with the same ``type`` and ``state_key`` in the room state. + example: "my_key" + content: + type: object + description: The content of the event + prev_events: + type: array + description: |- + Event IDs and hashes of the most recent events in the room that the homeserver was aware + of when it made this event + items: + type: array + maxItems: 2 + minItems: 2 + items: + - type: string + title: Event ID + example: "$abc123:matrix.org" + - type: object + title: Event Hash + example: { + "sha256": "abase64encodedsha256hashshouldbe43byteslong" + } + properties: + sha256: + type: string + description: The event hash + example: abase64encodedsha256hashshouldbe43byteslong + required: ['sha256'] + depth: + type: integer + description: The maximum depth of the ``prev_events``, plus one + example: 12 + auth_events: + type: array + description: Event IDs and hashes for the "auth events" of this event + items: + type: array + maxItems: 2 + minItems: 2 + items: + - type: string + title: Event ID + example: "$abc123:matrix.org" + - type: object + title: Event Hash + example: { + "sha256": "abase64encodedsha256hashshouldbe43byteslong" + } + properties: + sha256: + type: string + description: The event hash + example: abase64encodedsha256hashshouldbe43byteslong + required: ['sha256'] + hashes: + type: object + title: Event Hash + description: Hashes of the PDU, following the algorithm specified in `Signing Events`_ + example: { + "sha256": "thishashcoversallfieldsincasethisisredacted" + } + properties: + sha256: + type: string + description: The hash + example: thishashcoversallfieldsincasethisisredacted + required: ['sha256'] + signatures: + type: object + description: |- + Signatures of the redacted PDU, following the algorithm specified in `Signing Events`_ + example: { + "example.com": { + "ed25519:key_version:": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus" + } + } + additionalProperties: + type: object + title: Server Signatures + additionalProperties: + type: string + redacts: + type: string + description: For redaction events, the ID of the event being redacted + example: "$def456:matrix.org" + unsigned: + type: object + description: Additional data added by the origin server but not covered by the ``signatures`` +required: + - room_id + - sender + - origin + - origin_server_ts + - type + - content + - prev_events + - depth + - auth_events + - hashes + - signatures diff --git a/api/server-server/definitions/transaction.yaml b/api/server-server/definitions/transaction.yaml new file mode 100644 index 00000000..630d2ba3 --- /dev/null +++ b/api/server-server/definitions/transaction.yaml @@ -0,0 +1,35 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +title: Transaction +description: Transaction +example: + $ref: "../examples/transaction.json" +properties: + origin: + type: string + description: |- + The ``server_name`` of hoemserver sending this transaction + example: "example.org" + origin_server_ts: + type: integer + format: int64 + description: Timestamp in milliseconds on originating homeserver when this transaction started + example: 1234567890 + pdus: + type: array + description: List of persistent updates to rooms + items: + $ref: "pdu.yaml" +required: ['origin', 'origin_server_ts', 'pdus'] \ No newline at end of file diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml new file mode 100644 index 00000000..055883c2 --- /dev/null +++ b/api/server-server/events.yaml @@ -0,0 +1,118 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Events API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/state/{roomId}": + get: + summary: Get all the state of a given room + description: |- + Retrieves a snapshot of the entire current state of the given room. + operationId: getRoomState + parameters: + - in: path + name: roomId + type: string + description: The room ID to get state for + required: true + x-example: "!abc123:matrix.org" + responses: + 200: + description: The room state for the room (kept under ``pdus``) + schema: + $ref: "definitions/transaction.yaml" + "/event/{eventId}": + get: + summary: Get a single event + description: |- + Retrieves a single event + operationId: getEvent + parameters: + - in: path + name: eventId + type: string + description: The event ID to get + required: true + x-example: "$abc123:matrix.org" + responses: + 200: + description: A transaction containing a single PDU which is the event requested + schema: + $ref: "definitions/transaction.yaml" + "/backfill/{roomId}": + get: + summary: Retrieves the events which precede the given event + description: |- + Retreives a sliding-window history of previous PDUs that occurred on the given room. + Starting from the PDU ID(s) given in the ``v`` argument, the PDUs that preceded it + are retrived, up to the total number given by the ``limit``. + operationId: backfillRoom + parameters: + - in: path + name: roomId + type: string + description: The room ID to backfill + required: true + x-example: "!abc123:matrix.org" + - in: query + name: v + type: string # TODO: The description says this is plural - figure out how to specify multiple, and spec it + description: The event ID to backfill from + required: true + x-example: "$abc123:matrix.org" + - in: query + name: limit + type: integer + description: The maximum number of events to retrieve + required: true # TODO: Verify + x-example: 10 + responses: + 200: + description: A transaction containing the PDUs that preceded the given event(s). + schema: + $ref: "definitions/transaction.yaml" + "/pull": + get: + summary: Stream events later than a given point in history + description: |- + Retrieves all of the transactions later than any version given by the ``v`` arguments. + operationId: pull + parameters: + - in: query + name: v + type: string # TODO: The description says this is plural - figure out how to specify multiple, and spec it + description: The event ID to backfill from + required: true + x-example: "$abc123:matrix.org" + - in: query + name: origin + type: string + description: The origin # TODO: What is this actually? + required: true # TODO: Verify + x-example: "matrix.org" + responses: + 200: + # TODO: This could do with a better description + description: A transaction containing multiple PDUs + schema: + $ref: "definitions/transaction.yaml" \ No newline at end of file diff --git a/api/server-server/examples/edu.json b/api/server-server/examples/edu.json new file mode 100644 index 00000000..95a7b55d --- /dev/null +++ b/api/server-server/examples/edu.json @@ -0,0 +1,8 @@ +{ + "edu_type": "m.presence", + "origin": "blue", + "destination": "orange", + "content": { + "key": "value" + } +} \ No newline at end of file diff --git a/api/server-server/examples/full_transaction.json b/api/server-server/examples/full_transaction.json new file mode 100644 index 00000000..c453d6ed --- /dev/null +++ b/api/server-server/examples/full_transaction.json @@ -0,0 +1,6 @@ +{ + "origin": "matrix.org", + "origin_server_ts": 1234567890, + "pdus": [{"$ref": "pdu.json"}], + "edus": [{"$ref": "edu.json"}] +} \ No newline at end of file diff --git a/api/server-server/examples/pdu.json b/api/server-server/examples/pdu.json new file mode 100644 index 00000000..531f4a37 --- /dev/null +++ b/api/server-server/examples/pdu.json @@ -0,0 +1,25 @@ +{ + "room_id": "!UcYsUzyxTGDxLBEvLy:example.org", + "sender": "@alice:example.com", + "origin": "example.com", + "event_id": "$a4ecee13e2accdadf56c1025:example.com", + "origin_server_ts": 1404838188000, + "type": "m.room.message", + "prev_events": [ + [ + "$af232176:example.org", + {"sha256": "abase64encodedsha256hashshouldbe43byteslong"} + ] + ], + "hashes": { + "sha256": "thishashcoversallfieldsincasethisisredacted" + }, + "signatures": { + "example.com": { + "ed25519:key_version:": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus" + } + }, + "content": { + "key": "value" + } +} \ No newline at end of file diff --git a/api/server-server/examples/transaction.json b/api/server-server/examples/transaction.json new file mode 100644 index 00000000..bd8ac3dc --- /dev/null +++ b/api/server-server/examples/transaction.json @@ -0,0 +1,5 @@ +{ + "origin": "matrix.org", + "origin_server_ts": 1234567890, + "pdus": [{"$ref": "pdu.json"}] +} \ No newline at end of file diff --git a/api/server-server/transactions.yaml b/api/server-server/transactions.yaml new file mode 100644 index 00000000..389eecc7 --- /dev/null +++ b/api/server-server/transactions.yaml @@ -0,0 +1,51 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Transaction API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/send/{txnId}": + put: + summary: Send a transaction + description: |- + Push messages representing live activity to another server. The destination name + will be set to that of the receiving server itself. Each embedded PDU in the + transaction body will be processed. + operationId: sendTransaction + parameters: + - in: path + name: txnId + type: string + # TODO: "Overrides any ID given in the JSON body" - What does this mean? + description: |- + The transaction ID. Overrides any ID given in the JSON body. + required: true + x-example: TODO # No examples in the spec so far + - in: body + name: body + type: object + schema: + $ref: "definitions/full_transaction.yaml" + responses: + 200: + # TODO: Spec this (and figure out what it is) + description: TODO diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index dac183d1..7819dbf9 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -292,10 +292,19 @@ def process_data_type(prop, required=False, enforce_title=True): is_object = True elif prop_type == "array": - nested = process_data_type(prop["items"]) - prop_title = "[%s]" % nested["title"] - tables = nested["tables"] - enum_desc = nested["enum_desc"] + items = prop["items"] + if isinstance(items, list): + prop_title = "[" + for i in items: + nested = process_data_type(i) + tables.extend(nested['tables']) + prop_title = "%s%s, " % (prop_title, nested['title']) + prop_title = prop_title[:-2] + "]" + else: + nested = process_data_type(prop["items"]) + prop_title = "[%s]" % nested["title"] + tables = nested["tables"] + enum_desc = nested["enum_desc"] else: prop_title = prop_type @@ -505,7 +514,13 @@ class MatrixUnits(Units): if val_type == "array": items = param.get("items") if items: - val_type = "[%s]" % items.get("type") + if isinstance(items, list): + val_type = "[" + for i in items: + val_type += "%s, " % items.get("type") + val_type = val_type[:-2] + "]" + else: + val_type = "[%s]" % items.get("type") if param.get("enum"): val_type = "enum" diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f2b3204a..7b66584d 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -360,6 +360,10 @@ the destination. PDU Fields ~~~~~~~~~~ +.. TODO-spec + + Figure out how to embed swagger definitions in here (or improve the section) + ==================== ================== ======================================= Key Type Description ==================== ================== ======================================= @@ -749,57 +753,10 @@ All these URLs are name-spaced within a prefix of:: /_matrix/federation/v1/... -For active pushing of messages representing live activity "as it happens":: - PUT .../send// - Body: JSON encoding of a single Transaction - Response: TODO-doc - -The transaction_id path argument will override any ID given in the JSON body. -The destination name will be set to that of the receiving server itself. Each -embedded PDU in the transaction body will be processed. - - -To fetch all the state of a given room:: - - GET .../state// - Response: JSON encoding of a single Transaction containing multiple PDUs - -Retrieves a snapshot of the entire current state of the given room. The -response will contain a single Transaction, inside which will be a list of PDUs -that encode the state. - - -To fetch a particular event:: - - GET .../event// - Response: JSON encoding of a partial Transaction containing the event - -Retrieves a single event. The response will contain a partial Transaction, -having just the ``origin``, ``origin_server_ts`` and ``pdus`` fields; the -event will be encoded as the only PDU in the ``pdus`` list. - - -To backfill events on a given room:: - - GET .../backfill// - Query args: v, limit - Response: JSON encoding of a single Transaction containing multiple PDUs - -Retrieves a sliding-window history of previous PDUs that occurred on the given -room. Starting from the PDU ID(s) given in the "v" argument, the PDUs that -preceded it are retrieved, up to a total number given by the "limit" argument. - - -To stream all the events:: - - GET .../pull/ - Query args: origin, v - Response: JSON encoding of a single Transaction consisting of multiple PDUs - -Retrieves all of the transactions later than any version given by the "v" -arguments. +{{transactions_ss_http_api}} +{{events_ss_http_api}} To make a query:: From 114bcf1a2e1d612c15a735612e8a8f0ab9fb3b4f Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 6 Jul 2018 10:28:57 +0100 Subject: [PATCH 181/308] Use $ref, clean up, fix errors, AS is now a C-S module. --- .../application_service.yaml | 192 +---- .../definitions/schema/location.yaml | 30 + .../definitions/schema/location_batch.yaml | 17 + .../definitions/schema/protocol.yaml | 79 ++ .../definitions/schema/protocol_metadata.yaml | 66 ++ .../definitions/schema/user.yaml | 31 + .../definitions/schema/user_batch.yaml | 17 + api/client-server/third_party_lookup.yaml | 209 +---- specification/feature_profiles.rst | 2 + .../application_services.rst} | 2 + specification/proposals.rst | 770 +----------------- specification/targets.yaml | 2 +- 12 files changed, 323 insertions(+), 1094 deletions(-) create mode 100644 api/application-service/definitions/schema/location.yaml create mode 100644 api/application-service/definitions/schema/location_batch.yaml create mode 100644 api/application-service/definitions/schema/protocol.yaml create mode 100644 api/application-service/definitions/schema/protocol_metadata.yaml create mode 100644 api/application-service/definitions/schema/user.yaml create mode 100644 api/application-service/definitions/schema/user_batch.yaml rename specification/{third_party_lookup.rst => modules/application_services.rst} (97%) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index 86411ffa..b68cbe18 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -131,7 +131,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -140,7 +140,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 404: description: |- The application service indicates that this room alias does not exist. @@ -150,7 +150,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - type: object + $ref: ../client-server/definitions/error.yaml "/users/{userId}": get: summary: Query if a user should exist on the application service. @@ -187,7 +187,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -196,7 +196,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 404: description: |- The application service indicates that this user does not exist. @@ -206,7 +206,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - type: object + $ref: ../client-server/definitions/error.yaml "/_matrix/app/unstable/thirdparty/protocol/{protocol}": get: summary: Retrieve metadata about a specific protocol that the application service supports. @@ -219,44 +219,14 @@ paths: - in: path name: protocol type: string - description: |- - The name of the protocol. + description: The protocol ID. required: true x-example: "irc" responses: 200: description: The protocol was found and metadata returned. - examples: - application/json: { - "user_fields": ["network", "nickname"], - "location_fields": ["network", "channel"], - "icon": "mxc://example.org/aBcDeFgH", - "field_types": { - "network": { - "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", - "placeholder": "irc.example.org" - }, - "nickname": { - "regexp": "[^\\s]+", - "placeholder": "username" - }, - "channel": { - "regexp": "#[^\\s]+", - "placeholder": "#foobar" - } - }, - "instances": [ - { - "desc": "Freenode", - "icon": "mxc://example.org/JkLmNoPq", - "fields": { - "network": "freenode.net", - } - } - ] - } schema: - type: object + $ref: definitions/schema/protocol_metadata.yaml 401: description: |- The homeserver has not supplied credentials to the application service. @@ -266,7 +236,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -275,7 +245,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 404: description: No protocol was found with the given path. examples: @@ -283,20 +253,20 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - type: object + $ref: ../client-server/definitions/error.yaml "/_matrix/app/unstable/thirdparty/user/{protocol}": get: - summary: Retrieve the Matrix ID of a corresponding third party user. + summary: Retrieve the Matrix User ID of a corresponding third party user. description: |- - This API is called by the homeserver in order to retrieve a Matrix ID linked - to a user on the external service, given a set of user parameters. + This API is called by the homeserver in order to retrieve a Matrix + User ID linked to a user on the third party network, given a set of + user parameters. operationId: queryUserByProtocol parameters: - in: path name: protocol type: string - description: |- - The name of the protocol. + description: The protocol ID. required: true x-example: irc - in: query @@ -307,35 +277,9 @@ paths: service to help identify the user. responses: 200: - description: The Matrix IDs found with the given parameters. - examples: - application/json: [{ - "userid": "@_gitter_jim:matrix.org", - "protocol": "gitter", - "fields": { - "user": "jim" - } - }] + description: The Matrix User IDs found with the given parameters. schema: - type: array - description: Matched users. - items: - type: object - title: User - schema: - userid: - type: string - description: The Matrix ID of the matched user. - protocol: - type: string - description: The third party protocol. - fields: - type: object - description: The third party network fields used to identify this user. - properties: - user: - type: string - description: An example field, in this case the username for a Gitter user. + $ref: definitions/schema/user_batch.yaml 401: description: |- The homeserver has not supplied credentials to the application service. @@ -345,7 +289,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -354,7 +298,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 404: description: No users were found with the given parameters. examples: @@ -362,27 +306,20 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - type: object + $ref: ../client-server/definitions/error.yaml "/_matrix/app/unstable/thirdparty/location/{protocol}": get: summary: Retreive Matrix-side portal rooms leading to a third party location. description: |- - Requesting this endpoint with a valid protocol name results in a list - of successful mapping results in a JSON array. Each result contains - objects to represent the Matrix room or rooms that represent a portal - to this third party network. Each has the Matrix room alias string, - an identifier for the particular third party network protocol, and an - object containing the network-specific fields that comprise this - identifier. It should attempt to canonicalise the identifier as much - as reasonably possible given the network type. + Retrieve a list of Matrix portal rooms that lead to the matched third party location. operationId: queryLocationByProtocol parameters: - in: path name: protocol type: string - description: The protocol used to communicate to the third party network. + description: The protocol ID. required: true - x-example: "irc" + x-example: irc - in: query name: field1, field2... type: string @@ -392,23 +329,8 @@ paths: responses: 200: description: At least one portal room was found. - examples: - application/json: [{ - "alias": "#freenode_#matrix:matrix.org", - "protocol": "irc", - "fields": { - "network": "freenode", - "channel": "#matrix" - } - }] schema: - type: array - description: |- - Array of portal rooms leading to the requested third party - location. - items: - type: object - title: Portal Room + $ref: definitions/schema/location_batch.yaml 401: description: |- The homeserver has not supplied credentials to the application service. @@ -418,7 +340,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -427,7 +349,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 404: description: No mappings were found with the given parameters. examples: @@ -435,7 +357,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - type: object + $ref: ../client-server/definitions/error.yaml "/_matrix/app/unstable/thirdparty/location": get: summary: Reverse-lookup third party locations given a Matrix room alias. @@ -444,30 +366,16 @@ paths: alias. operationId: queryLocationByAlias parameters: - - in: query - name: alias - type: string - description: The Matrix room alias to look up. + - in: query + name: alias + type: string + description: The Matrix room alias to look up. responses: 200: description: |- - All found third party locations. Same response format as the - forward lookup response. - examples: - application/json: [{ - "alias": "#freenode_#matrix:matrix.org", - "protocol": "irc", - "fields": { - "network": "freenode", - "channel": "#matrix" - } - }] + All found third party locations. schema: - type: array - description: Matched third party locations. - items: - type: object - title: Location + $ref: definitions/schema/location_batch.yaml 401: description: |- The homeserver has not supplied credentials to the application service. @@ -477,7 +385,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -486,7 +394,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 404: description: No mappings were found with the given parameters. examples: @@ -494,36 +402,24 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - type: object + $ref: ../client-server/definitions/error.yaml "/_matrix/app/unstable/thirdparty/user": get: - summary: Reverse-lookup third party users given a Matrix ID. + summary: Reverse-lookup third party users given a Matrix User ID. description: |- - Retreive an array of third party users from a Matrix ID. + Retreive an array of third party users from a Matrix User ID. operationId: queryUserByID parameters: - in: query name: userid type: string - description: The Matrix ID to look up. + description: The Matrix User ID to look up. responses: 200: description: |- An array of third party users. - examples: - application/json: [{ - "userid": "@_gitter_jim:matrix.org", - "protocol": "gitter", - "fields": { - "user": "jim" - } - }] schema: - type: array - description: Matched third party users - items: - type: object - title: User + $ref: definitions/schema/user_batch.yaml 401: description: |- The homeserver has not supplied credentials to the application service. @@ -533,7 +429,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -542,7 +438,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - type: object + $ref: ../client-server/definitions/error.yaml 404: description: No mappings were found with the given parameters. examples: @@ -550,4 +446,4 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - type: object \ No newline at end of file + $ref: ../client-server/definitions/error.yaml \ No newline at end of file diff --git a/api/application-service/definitions/schema/location.yaml b/api/application-service/definitions/schema/location.yaml new file mode 100644 index 00000000..4967ef61 --- /dev/null +++ b/api/application-service/definitions/schema/location.yaml @@ -0,0 +1,30 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +properties: + alias: + description: An alias for a matrix room. + type: string + example: "#freenode_#matrix:matrix.org" + protocol: + description: The protocol ID that the third party location is a part of. + type: string + example: irc + fields: + description: Information used to identify this third party location. + type: object + example: + "network": "freenode" + "channel": "#matrix" +title: Location +type: object \ No newline at end of file diff --git a/api/application-service/definitions/schema/location_batch.yaml b/api/application-service/definitions/schema/location_batch.yaml new file mode 100644 index 00000000..3f6de9df --- /dev/null +++ b/api/application-service/definitions/schema/location_batch.yaml @@ -0,0 +1,17 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: array +description: List of matched third party locations. +items: + $ref: location.yaml diff --git a/api/application-service/definitions/schema/protocol.yaml b/api/application-service/definitions/schema/protocol.yaml new file mode 100644 index 00000000..231e8288 --- /dev/null +++ b/api/application-service/definitions/schema/protocol.yaml @@ -0,0 +1,79 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +properties: + user_fields: + description: Fields used to identify a third party user. + type: array + items: + type: string + description: Field used to identify a third party user. + example: ["network", "nickname"] + location_fields: + description: Fields used to identify a third party location. + type: array + items: + type: string + description: Field used to identify a third party location. + example: ["network", "channel"] + icon: + description: An icon representing the third party protocol. + type: string + example: "mxc://example.org/aBcDeFgH" + field_types: + title: Field Types + description: All location or user fields should have an entry here. + type: object + properties: + fieldname: + title: Field Type + description: Definition of valid values for a field. + type: object + properties: + regexp: + description: A regular expression for validation of a field's value. + type: string + placeholder: + description: An placeholder serving as a valid example of the field value. + type: string + example: { + "network": { + "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", + "placeholder": "irc.example.org" + }, + "nickname": { + "regexp": "[^\\s#]+", + "placeholder": "username" + }, + "channel": { + "regexp": "#[^\\s]+", + "placeholder": "#foobar" + } + } + instances: + description: |- + A list of objects representing independent instances of configuration. + For instance multiple networks on IRC if multiple are bridged by the + same bridge. + type: array + items: + type: object + example: { + "desc": "Freenode", + "icon": "mxc://example.org/JkLmNoPq", + "fields": { + "network": "freenode.net", + } + } +title: Protocol +type: object \ No newline at end of file diff --git a/api/application-service/definitions/schema/protocol_metadata.yaml b/api/application-service/definitions/schema/protocol_metadata.yaml new file mode 100644 index 00000000..72264060 --- /dev/null +++ b/api/application-service/definitions/schema/protocol_metadata.yaml @@ -0,0 +1,66 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +description: Dictionary of supported third party protocols. +example: { + "irc": { + "user_fields": ["network", "nickname"], + "location_fields": ["network", "channel"], + "icon": "mxc://example.org/aBcDeFgH", + "field_types": { + "network": { + "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", + "placeholder": "irc.example.org" + }, + "nickname": { + "regexp": "[^\\s]+", + "placeholder": "username" + }, + "channel": { + "regexp": "#[^\\s]+", + "placeholder": "#foobar" + } + }, + "instances": [ + { + "desc": "Freenode", + "icon": "mxc://example.org/JkLmNoPq", + "fields": { + "network": "freenode.net", + } + } + ] + }, + "gitter": { + "user_fields": ["username"], + "location_fields": ["room"], + "field_types": { + "username": { + "regexp": "@[^\\s]+", + "placeholder": "@username" + }, + "room": { + "regexp": "[^\\s]+\\/[^\\s]+", + "placeholder": "matrix-org/matrix-doc" + } + }, + "instances": [ + { + "desc": "Gitter", + "icon": "mxc://example.org/zXyWvUt", + "fields": {} + } + ] + } +} \ No newline at end of file diff --git a/api/application-service/definitions/schema/user.yaml b/api/application-service/definitions/schema/user.yaml new file mode 100644 index 00000000..5f8d0460 --- /dev/null +++ b/api/application-service/definitions/schema/user.yaml @@ -0,0 +1,31 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: Change userid to user_id as a breaking change +properties: + userid: + description: A Matrix User ID represting a third party user. + type: string + example: "@_gitter_jim:matrix.org" + protocol: + description: The protocol ID that the third party location is a part of. + type: string + example: gitter + fields: + description: Information used to identify this third party location. + type: object + example: + "user": "jim" +title: Location +type: object \ No newline at end of file diff --git a/api/application-service/definitions/schema/user_batch.yaml b/api/application-service/definitions/schema/user_batch.yaml new file mode 100644 index 00000000..3653feb4 --- /dev/null +++ b/api/application-service/definitions/schema/user_batch.yaml @@ -0,0 +1,17 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: array +description: List of matched third party users. +items: + $ref: user.yaml diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 343cf2b7..86389603 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -24,6 +24,8 @@ consumes: - application/json produces: - application/json +securityDefinitions: + $ref: definitions/security.yaml paths: "/thirdparty/protocols": get: @@ -36,60 +38,8 @@ paths: responses: 200: description: The protocols supported by the homeserver. - examples: - application/json: { - "irc": { - "user_fields": ["network", "nickname"], - "location_fields": ["network", "channel"], - "icon": "mxc://example.org/aBcDeFgH", - "field_types": { - "network": { - "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", - "placeholder": "irc.example.org" - }, - "nickname": { - "regexp": "[^\\s]+", - "placeholder": "username" - }, - "channel": { - "regexp": "#[^\\s]+", - "placeholder": "#foobar" - } - }, - "instances": [ - { - "desc": "Freenode", - "icon": "mxc://example.org/JkLmNoPq", - "fields": { - "network": "freenode.net", - } - } - ] - }, - "gitter": { - "user_fields": ["username"], - "location_fields": ["room"], - "field_types": { - "username": { - "regexp": "@[^\\s]+", - "placeholder": "@username" - }, - "room": { - "regexp": "[^\\s]+\\/[^\\s]+", - "placeholder": "matrix-org/matrix-doc" - } - }, - "instances": [ - { - "desc": "Gitter", - "icon": "mxc://example.org/zXyWvUt", - "fields": {} - } - ] - } - } schema: - type: object + $ref: ../application-service/definitions/schema/protocol_metadata.yaml "/thirdparty/protocol/{protocol}": get: summary: Retrieve metadata about a specific protocol that the homeserver supports. @@ -107,45 +57,16 @@ paths: responses: 200: description: The protocol was found and metadata returned. - examples: - application/json: { - "user_fields": ["network", "nickname"], - "location_fields": ["network", "channel"], - "icon": "mxc://example.org/aBcDeFgH", - "field_types": { - "network": { - "regexp": "([a-z0-9]+\\.)*[a-z0-9]+", - "placeholder": "irc.example.org" - }, - "nickname": { - "regexp": "[^\\s#]+", - "placeholder": "username" - }, - "channel": { - "regexp": "#[^\\s]+", - "placeholder": "#foobar" - } - }, - "instances": [ - { - "desc": "Freenode", - "icon": "mxc://example.org/JkLmNoPq", - "fields": { - "network": "freenode.net", - } - } - ] - } schema: - type: object + $ref: ../application-service/definitions/schema/protocol.yaml 404: - description: The protocol is unknown + description: The protocol is unknown. examples: application/json: { "errcode": "M_NOT_FOUND" } schema: - type: object + $ref: definitions/error.yaml "/thirdparty/location/{protocol}": get: summary: Retreive Matrix-side portals rooms leading to a third party location. @@ -165,9 +86,9 @@ paths: type: string description: The protocol used to communicate to the third party network. required: true - x-example: "irc" + x-example: irc - in: query - name: field1, field2... + name: searchFields type: string description: |- One or more custom fields to help identify the third party @@ -175,36 +96,21 @@ paths: responses: 200: description: At least one portal room was found. - examples: - application/json: [{ - "alias": "#freenode_#matrix:matrix.org", - "protocol": "irc", - "fields": { - "network": "freenode", - "channel": "#matrix" - } - }] schema: - type: array - description: |- - Array of portal rooms leading to the requested third party - location. - items: - type: object - title: Portal Room + $ref: ../application-service/definitions/schema/location_batch.yaml 404: - description: The Matrix room alias was not found + description: No portal rooms were found. examples: application/json: { "errcode": "M_NOT_FOUND" } schema: - type: object + $ref: definitions/error.yaml "/thirdparty/user/{protocol}": get: - summary: Retrieve the Matrix ID of a corresponding third party user. + summary: Retrieve the Matrix User ID of a corresponding third party user. description: |- - Retrieve a Matrix ID linked to a user on the third party service, given + Retrieve a Matrix User ID linked to a user on the third party service, given a set of user parameters. operationId: queryUserByProtocol parameters: @@ -222,43 +128,16 @@ paths: One or more custom fields that are passed to the AS to help identify the user. responses: 200: - description: The Matrix IDs found with the given parameters. - examples: - application/json: [{ - "userid": "@_gitter_jim:matrix.org", - "protocol": "gitter", - "fields": { - "user": "jim" - } - }] - schema: - type: array - description: Matched users. - items: - type: object - title: User - schema: - userid: - type: string - description: The Matrix ID of the matched user. - protocol: - type: string - description: The third party protocol. - fields: - type: object - description: The third party network fields used to identify this user. - properties: - user: - type: string - description: An example field, in this case the username for a Gitter user. + description: The Matrix User IDs found with the given parameters. + $ref: ../application-service/definitions/schema/user_batch.yaml 404: - description: The Matrix ID was not found + description: The Matrix User ID was not found examples: application/json: { "errcode": "M_NOT_FOUND" } schema: - type: object + $ref: definitions/error.yaml "/thirdparty/location": get: summary: Reverse-lookup third party locations given a Matrix room alias. @@ -267,30 +146,17 @@ paths: alias. operationId: queryLocationByAlias parameters: - - in: path - name: alias - type: string - description: The Matrix room alias to look up. + - in: path + name: alias + type: string + description: The Matrix room alias to look up. + required: true responses: 200: description: |- - All found third party locations. Same response format as the - forward lookup response. - examples: - application/json: [{ - "alias": "#freenode_#matrix:matrix.org", - "protocol": "irc", - "fields": { - "network": "freenode", - "channel": "#matrix" - } - }] + All found third party locations. schema: - type: array - description: Matched third party locations. - items: - type: object - title: Location + $ref: ../application-service/definitions/schema/location_batch.yaml 404: description: The Matrix room alias was not found examples: @@ -298,41 +164,30 @@ paths: "errcode": "M_NOT_FOUND" } schema: - type: object + $ref: definitions/error.yaml "/thirdparty/user": get: - summary: Reverse-lookup third party users given a Matrix ID. + summary: Reverse-lookup third party users given a Matrix User ID. description: |- - Retreive an array of third party users from a Matrix ID. + Retreive an array of third party users from a Matrix User ID. operationId: queryUserByID parameters: - in: path name: userid type: string - description: The Matrix ID to look up. + description: The Matrix User ID to look up. + required: true responses: 200: description: |- An array of third party users. - examples: - application/json: [{ - "userid": "@_gitter_jim:matrix.org", - "protocol": "gitter", - "fields": { - "user": "jim" - } - }] schema: - type: array - description: Matched third party users - items: - type: object - title: User + $ref: ../application-service/definitions/schema/user_batch.yaml 404: - description: The Matrix ID was not found + description: The Matrix User ID was not found examples: application/json: { "errcode": "M_NOT_FOUND" } schema: - type: object \ No newline at end of file + $ref: definitions/error.yaml \ No newline at end of file diff --git a/specification/feature_profiles.rst b/specification/feature_profiles.rst index 7fc9696d..97d0de0e 100644 --- a/specification/feature_profiles.rst +++ b/specification/feature_profiles.rst @@ -42,6 +42,7 @@ Summary `Server Side Search`_ Optional Optional Optional Optional Optional `Server Administration`_ Optional Optional Optional Optional Optional `Event Context`_ Optional Optional Optional Optional Optional + `Application Services`_ Optional Optional Optional Optional Optional ===================================== ========== ========== ========== ========== ========== *Please see each module for more details on what clients need to implement.* @@ -57,6 +58,7 @@ Summary .. _Server Side Search: `module:search`_ .. _Server Administration: `module:admin`_ .. _Event Context: `module:event-context`_ +.. _Application Services: `module:application-services`_ Clients ------- diff --git a/specification/third_party_lookup.rst b/specification/modules/application_services.rst similarity index 97% rename from specification/third_party_lookup.rst rename to specification/modules/application_services.rst index 6325dca0..52e35dc8 100644 --- a/specification/third_party_lookup.rst +++ b/specification/modules/application_services.rst @@ -1,6 +1,8 @@ Application Services ==================== +.. _module:application-services: + An application service is a separate program that interacts with a homeserver and provides various bits of functionality that would otherwise not make sense to include directly in the homeserver. This ranges from bots, which can diff --git a/specification/proposals.rst b/specification/proposals.rst index d04edfa8..371850ab 100644 --- a/specification/proposals.rst +++ b/specification/proposals.rst @@ -1,772 +1,6 @@ Tables of Tracked Proposals --------------------------- -proposal-wip -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +This file is autogenerated by a jenkins build process -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC455 `_ - - Do we want to specify a matrix:// URI scheme for rooms? (SPEC-5) - - 2014-09-15 - - 2018-05-15 - - `455-1 `_ - - `@KitsuneRal`_ - - None - - - * - `MSC586 `_ - - Federation API for canonicalising MXIDs - - 2014-10-27 - - 2018-05-15 - - `586-1 `_ - - `@ara4n`_ - - None - - - * - `MSC489 `_ - - Extensible Profiles. (SPEC-93) - - 2015-01-19 - - 2018-05-15 - - `489-1 `_ - - `@erikjohnston`_ - - None - - - * - `MSC1196 `_ - - Matrix Escape Hatch (Versioned Rooms) - - 2015-10-22 - - 2018-05-15 - - `1196-1 `_ - - `@leonerd`_ - - None - - - * - `MSC1148 `_ - - Support for websockets - - 2015-11-16 - - 2018-06-04 - - `1148-1 `_, `1148-2 `_ - - `@richvdh`_, `@krombel`_ - - None - - - * - `MSC1238 `_ - - Push to Talk - - 2016-04-13 - - 2018-05-15 - - `1238-1 `_ - - `@aviraldg`_ - - None - - `PR#310`_ - * - `MSC1198 `_ - - Threading API - - 2016-05-23 - - 2018-05-15 - - `1198-1 `_ - - `@Half-Shot`_ - - None - - - * - `MSC1207 `_ - - Publishing Room Lists for 3rd party networks - - 2016-10-21 - - 2018-05-31 - - `1207-1 `_ - - `@erikjohnston`_ - - None - - - * - `MSC441 `_ - - Support for Reactions / Aggregations - - 2016-12-25 - - 2018-05-15 - - `441-1 `_ - - `@pik`_ - - `@ara4n`_ - - - * - `MSC1237 `_ - - Improving m.location with GeoJSON and accuracy - - 2017-05-19 - - 2018-05-15 - - `1237-1 `_ - - `@Half-Shot`_ - - None - - `PR#919`_ - * - `MSC971 `_ - - Add groups stuff to spec - - 2017-08-10 - - 2018-05-20 - - `971-1 `_, `971-2 `_, `971-3 `_ - - `@erikjohnston`_ - - None - - - * - `MSC1215 `_ - - Groups as Rooms - - 2017-10-17 - - 2018-05-15 - - `1215-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1217 `_ - - Visibility of groups to group members and their non-members - - 2017-10-30 - - 2018-05-15 - - `1217-1 `_ - - `@lampholder`_ - - None - - - * - `MSC1218 `_ - - Bridging group membership with 3rd party group sources - - 2017-11-15 - - 2018-05-15 - - `1218-1 `_ - - `@lukebarnard1`_ - - None - - - * - `MSC1219 `_ - - Proposal for storing megolm keys serverside - - 2017-11-23 - - 2018-05-15 - - `1219-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1221 `_ - - Improving Presence - - 2017-12-26 - - 2018-05-24 - - `1221-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1222 `_ - - Pushing updates about Groups (Communities) to clients - - 2018-01-02 - - 2018-05-24 - - `1222-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1225 `_ - - Extensible event types & fallback in Matrix - - 2018-02-18 - - 2018-05-15 - - `1225-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1206 `_ - - Proposal for improved bot support - - 2018-03-14 - - 2018-06-08 - - `1206-1 `_ - - `@turt2live`_ - - None - - - * - `MSC1228 `_ - - Removing MXIDs from events - - 2018-04-19 - - 2018-05-15 - - `1228-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1231 `_ - - Handling Consent for Privacy Policy - - 2018-05-02 - - 2018-05-15 - - `1231-1 `_ - - `@neilisfragile`_ - - None - - - * - `MSC1267 `_ - - Interactive Key Verification - - 2018-05-28 - - 2018-05-28 - - `1267-1 `_ - - `@uhoreg`_ - - None - - - * - `MSC1280 `_ - - Mechanisms for communicating erasure requests to bots and federated homeservers - - 2018-06-05 - - 2018-06-05 - - `1280-1 `_ - - `@richvdh`_ - - None - - - * - `MSC688 `_ - - Calculate room names server-side - - 2018-06-10 - - 2018-06-15 - - `688-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1324 `_ - - Custom protocol definitions for application services - - 2018-06-20 - - 2018-06-20 - - `1324-1 `_ - - `@anoadragon453`_ - - None - - - * - `MSC1323 `_ - - AS traffic rate-limiting - - 2018-06-20 - - 2018-07-03 - - `1323-1 `_ - - `@anoadragon453`_ - - None - - - * - `MSC1322 `_ - - Mechanism to communicate 3PID binding updates or deletions to homeservers - - 2018-06-20 - - 2018-06-20 - - `1322-1 `_ - - `@babolivier`_ - - None - - - - - -proposal-ready-for-review -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC1227 `_ - - Proposal for lazy-loading room members to improve initial sync speed and client RAM usage - - 2018-03-05 - - 2018-06-10 - - `1227-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1206 `_ - - Proposal for improved bot support - - 2018-03-14 - - 2018-06-08 - - `1206-1 `_ - - `@turt2live`_ - - None - - - * - `MSC1256 `_ - - Custom emoji and sticker packs in matrix - - 2018-05-23 - - 2018-05-24 - - `1256-1 `_ - - `@turt2live`_ - - None - - - * - `MSC1270 `_ - - Proposal Add /_matrix/media/v1/resolve_url to Client-Server API: download and preview urls in the clients despite CORS - - 2018-05-31 - - 2018-06-19 - - `1270-1 `_ - - `@oivoodoo`_ - - None - - - * - `MSC701 `_ - - Auth for content repo (and enforcing GDPR erasure) - - 2018-06-04 - - 2018-06-07 - - `701-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1304 `_ - - Proposal to simplify the auth rules of m.room.power_level events. - - 2018-06-13 - - 2018-06-14 - - `1304-1 `_ - - `@richvdh`_, `@ara4n`_ - - None - - - * - `MSC1301 `_ - - Proposal for improving authorization for the matrix profile API - - 2018-06-13 - - 2018-06-13 - - `1301-1 `_ - - `@turt2live`_ - - None - - - * - `MSC1309 `_ - - Proposal for an application service management API - - 2018-06-14 - - 2018-06-15 - - `1309-1 `_ - - `@turt2live`_ - - None - - - * - `MSC1308 `_ - - Proposal for speeding up review of simple spec changes - - 2018-06-14 - - 2018-06-24 - - - - `@ara4n`_ - - None - - - * - `MSC1306 `_ - - Proposal to filter out traffic to Appservices based on filters - - 2018-06-14 - - 2018-06-14 - - `1306-1 `_ - - `@Half-Shot`_ - - None - - - - - -proposal-in-review -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC1194 `_ - - A way for HSes to remove bindings from ISes (aka unbind) - - 2018-05-09 - - 2018-06-05 - - `1194-1 `_ - - `@dbkr`_ - - None - - - - - -proposal-passed-review -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC433 `_ - - Support for discovering API endpoints via .well-known URIs (SPEC-121) - - 2015-03-08 - - 2018-07-01 - - `433-1 `_, `433-2 `_ - - `@maxidor`_, `others`_ - - `@uhoreg`_ - - - * - `MSC1226 `_ - - State Reset mitigation proposal - - 2018-02-20 - - 2018-05-15 - - `1226-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1229 `_ - - Mitigating abuse of the event depth parameter over federation - - 2018-04-30 - - 2018-05-15 - - `1229-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1232 `_ - - Media limits API - - 2018-05-04 - - 2018-06-21 - - `1232-1 `_ - - `@Half-Shot`_ - - None - - `PR#1189`_ - * - `MSC1236 `_ - - Matrix Widget API v2 - - 2018-05-13 - - 2018-05-15 - - `1236-1 `_ - - `@rxl881`_ - - None - - - - - -spec-pr-missing -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC1200 `_ - - Configuration of E2E encryption in a room - - 2016-06-16 - - 2018-05-31 - - `1200-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1201 `_ - - Device Management API - - 2016-07-14 - - 2018-05-15 - - `1201-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1203 `_ - - 3rd Party Entity lookup API - - 2016-07-21 - - 2018-07-02 - - `1203-1 `_ - - `@leonerd`_ - - None - - - * - `MSC1208 `_ - - Encrypted attachment format - - 2016-10-26 - - 2018-05-15 - - `1208-1 `_ - - `@NegativeMjark`_ - - None - - - * - `MSC739 `_ - - Reporting inappropriate content in Matrix - - 2016-11-21 - - 2018-05-31 - - `739-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1211 `_ - - Megolm session export format - - 2017-01-03 - - 2018-05-15 - - `1211-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1212 `_ - - Device List Update Stream - - 2017-01-18 - - 2018-05-15 - - `1212-1 `_ - - `@richvdh`_ - - None - - - * - `MSC829 `_ - - Need to spec msisdn login API - - 2017-03-08 - - 2018-05-15 - - `829-1 `_ - - `@dbkr`_ - - None - - - * - `MSC855 `_ - - spec m.login.msisdn UI auth type - - 2017-03-24 - - 2018-05-15 - - `855-1 `_ - - `@dbkr`_ - - None - - - * - `MSC910 `_ - - Add new Read Marker API to docs - - 2017-05-08 - - 2018-05-15 - - - - `@lukebarnard1`_ - - None - - - * - `MSC1067 `_ - - Spec @mentions - - 2017-07-14 - - 2018-05-15 - - `1067-1 `_ - - `@lukebarnard1`_ - - None - - - * - `MSC1214 `_ - - Related Groups (i.e. flair) - - 2017-10-03 - - 2018-05-15 - - `1214-1 `_ - - `@lukebarnard1`_ - - None - - - * - `MSC1033 `_ - - Doc @room notifications - - 2017-10-23 - - 2018-05-31 - - - - `@dbkr`_ - - None - - - * - `MSC1183 `_ - - Document key-share requests - - 2018-04-30 - - 2018-05-31 - - `1183-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1230 `_ - - Temporary mitigation for depth parameter abuse - - 2018-05-01 - - 2018-05-15 - - `1230-1 `_ - - `@ara4n`_ - - None - - - * - `MSC1234 `_ - - Rich Replies format - - 2018-05-12 - - 2018-05-18 - - `1234-1 `_ - - `@t3chguy`_ - - None - - - - - -merged -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC1197 `_ - - Ignoring Users - - 2016-05-03 - - 2018-05-18 - - `1197-1 `_ - - `@erikjohnston`_ - - None - - `PR#1142`_ - * - `MSC1199 `_ - - Notifications API - - 2016-05-23 - - 2018-06-25 - - `1199-1 `_ - - `@dbkr`_ - - None - - - * - `MSC1204 `_ - - Access Token Semantics (refresh and macaroons) - aka Auth Sept 2016 Edition - - 2016-09-29 - - 2018-06-25 - - `1204-1 `_ - - `@richvdh`_ - - None - - - * - `MSC1205 `_ - - Proposal for multi-device deletion API - - 2016-10-12 - - 2018-06-25 - - `1205-1 `_ - - `@richvdh`_ - - None - - `PR#1239`_ - * - `MSC953 `_ - - Add /user_directory/search API - - 2017-05-31 - - 2018-05-10 - - `953-1 `_ - - `@erikjohnston`_ - - None - - - * - `MSC1233 `_ - - A proposal for organising spec proposals - - 2018-05-10 - - 2018-06-25 - - `1233-1 `_ - - `@ara4n`_ - - None - - `PR#1240`_ - - - -abandoned -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC531 `_ - - Homeservers as OAuth authorization endpoints (resource owners) (SPEC-206) - - 2015-07-25 - - 2018-05-15 - - `531-1 `_ - - `@Kegsay`_ - - None - - - * - `MSC1202 `_ - - Profile Personae - - 2016-07-15 - - 2018-05-15 - - `1202-1 `_ - - `@erikjohnston`_ - - None - - - * - `MSC1209 `_ - - Server Side Profile API - - 2016-11-01 - - 2018-05-15 - - `1209-1 `_ - - `@erikjohnston`_ - - None - - - * - `MSC1213 `_ - - Set defaults for m.federate - - 2017-04-10 - - 2018-05-18 - - `1213-1 `_ - - `@psaavedra`_ - - None - - - - - -obsolete -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: auto - :stub-columns: 1 - - * - MSC - - Proposal Title - - Creation Date - - Update Date - - Documentation - - Author - - Shepherd - - PRs - * - `MSC1223 `_ - - Replies event format - - 2018-02-01 - - 2018-05-15 - - `1223-1 `_ - - `@t3chguy`_ - - None - - - * - `MSC1224 `_ - - Replies - next steps - - 2018-02-03 - - 2018-05-15 - - `1224-1 `_ - - `@t3chguy`_ - - None - - - * - `MSC1235 `_ - - Proposal for Calendar Events - - 2018-02-06 - - 2018-05-15 - - `1235-1 `_ - - `@Half-Shot`_ - - None - - - * - `MSC1220 `_ - - Rich quoting proposal - - 2018-05-10 - - 2018-05-15 - - `1220-1 `_ - - `@t3chguy`_ - - None - - - - - - - -.. _@rxl881: https://github.com/rxl881 -.. _@turt2live: https://github.com/turt2live -.. _@erikjohnston: https://github.com/erikjohnston -.. _@anoadragon453: https://github.com/anoadragon453 -.. _@t3chguy: https://github.com/t3chguy -.. _@Kegsay: https://github.com/Kegsay -.. _@KitsuneRal: https://github.com/KitsuneRal -.. _@leonerd: https://github.com/leonerd -.. _@psaavedra: https://github.com/psaavedra -.. _@ara4n: https://github.com/ara4n -.. _@krombel: https://github.com/krombel -.. _@maxidor: https://github.com/maxidor -.. _@uhoreg: https://github.com/uhoreg -.. _@pik: https://github.com/pik -.. _@neilisfragile: https://github.com/neilisfragile -.. _@babolivier: https://github.com/babolivier -.. _@lukebarnard1: https://github.com/lukebarnard1 -.. _others: https://github.com/thers -.. _@Half-Shot: https://github.com/Half-Shot -.. _@aviraldg: https://github.com/aviraldg -.. _@oivoodoo: https://github.com/oivoodoo -.. _@richvdh: https://github.com/richvdh -.. _@NegativeMjark: https://github.com/NegativeMjark -.. _@lampholder: https://github.com/lampholder -.. _@dbkr: https://github.com/dbkr -.. _PR#1189: https://github.com/matrix-org/matrix-doc/pull/1189 -.. _PR#310: https://github.com/matrix-org/matrix-doc/pull/310 -.. _PR#1142: https://github.com/matrix-org/matrix-doc/pull/1142 -.. _PR#1239: https://github.com/matrix-org/matrix-doc/pull/1239 -.. _PR#1240: https://github.com/matrix-org/matrix-doc/pull/1240 -.. _PR#919: https://github.com/matrix-org/matrix-doc/pull/919 \ No newline at end of file +View the current live version `at https://matrix.org/docs/spec/proposals `_ diff --git a/specification/targets.yaml b/specification/targets.yaml index 96199d29..ba3e3299 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -12,7 +12,6 @@ targets: - { 1: modules.rst } - { 2: feature_profiles.rst } - { 2: "group:modules" } # reference a group of files - - { 1: third_party_lookup.rst } version_label: "%CLIENT_RELEASE_LABEL%" application_service: files: @@ -69,6 +68,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/ignore_users.rst - modules/stickers.rst - modules/report_content.rst + - modules/application_services.rst title_styles: ["=", "-", "~", "+", "^", "`", "@", ":"] From 9bde78ac288b198ed0af9514f0aa51b7f4d68bf4 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 12 Jul 2018 14:41:24 +0100 Subject: [PATCH 182/308] $ref needs to be under a schema: --- api/client-server/third_party_lookup.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 86389603..848d92d1 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -129,7 +129,8 @@ paths: responses: 200: description: The Matrix User IDs found with the given parameters. - $ref: ../application-service/definitions/schema/user_batch.yaml + schema: + $ref: ../application-service/definitions/schema/user_batch.yaml 404: description: The Matrix User ID was not found examples: From 89998872452c79be9efdf60b6d0936b9069b8855 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 12 Jul 2018 14:44:48 +0100 Subject: [PATCH 183/308] Path -> query params --- api/client-server/third_party_lookup.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 848d92d1..2fcf790b 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -147,7 +147,7 @@ paths: alias. operationId: queryLocationByAlias parameters: - - in: path + - in: query name: alias type: string description: The Matrix room alias to look up. @@ -173,7 +173,7 @@ paths: Retreive an array of third party users from a Matrix User ID. operationId: queryUserByID parameters: - - in: path + - in: query name: userid type: string description: The Matrix User ID to look up. From 16125f16f5cc1f1f02f96d20c3c11d7e6858ac06 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 09:28:30 -0600 Subject: [PATCH 184/308] Spec retry_after_ms on ratelimit errors --- api/client-server/definitions/errors/rate_limited.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/client-server/definitions/errors/rate_limited.yaml b/api/client-server/definitions/errors/rate_limited.yaml index 6b4932ee..aca82ce7 100644 --- a/api/client-server/definitions/errors/rate_limited.yaml +++ b/api/client-server/definitions/errors/rate_limited.yaml @@ -23,4 +23,10 @@ properties: type: string description: A human-readable error message. example: Too many requests + retry_after_ms: + type: integer + description: |- + The amount of time in milliseconds the client should wait + before trying the request again. + example: 2000 required: ["errcode"] \ No newline at end of file From 84e07a0c10276ac3b4d87586e8a25e74faad9506 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 09:31:50 -0600 Subject: [PATCH 185/308] Convert errors in presence.yaml --- api/client-server/presence.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/client-server/presence.yaml b/api/client-server/presence.yaml index 68b3ee56..91b75c6a 100644 --- a/api/client-server/presence.yaml +++ b/api/client-server/presence.yaml @@ -123,6 +123,8 @@ paths: description: |- There is no presence state for this user. This user may not exist or isn't exposing presence information to you. + schema: + "$ref": "definitions/errors/error.yaml" 403: description: You are not allowed to see this user's presence status. examples: @@ -131,7 +133,7 @@ paths: "error": "You are not allowed to see their presence" } schema: - "$ref": "definitions/error.yaml" + "$ref": "definitions/errors/error.yaml" tags: - Presence "/presence/list/{userId}": From 291a4dfc7612635f77b4d71a89f549111bdb7e49 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 12 Jul 2018 16:52:31 +0100 Subject: [PATCH 186/308] Third party lookup module. Fix wording --- .../application_service.yaml | 10 +++---- .../definitions/{schema => }/location.yaml | 0 .../{schema => }/location_batch.yaml | 0 .../definitions/{schema => }/protocol.yaml | 0 .../{schema => }/protocol_metadata.yaml | 0 .../definitions/{schema => }/user.yaml | 0 .../definitions/{schema => }/user_batch.yaml | 0 api/client-server/third_party_lookup.yaml | 12 ++++---- specification/feature_profiles.rst | 4 +-- .../modules/application_services.rst | 28 ------------------- .../modules/third_party_networks.rst | 20 +++++++++++++ specification/targets.yaml | 2 +- 12 files changed, 34 insertions(+), 42 deletions(-) rename api/application-service/definitions/{schema => }/location.yaml (100%) rename api/application-service/definitions/{schema => }/location_batch.yaml (100%) rename api/application-service/definitions/{schema => }/protocol.yaml (100%) rename api/application-service/definitions/{schema => }/protocol_metadata.yaml (100%) rename api/application-service/definitions/{schema => }/user.yaml (100%) rename api/application-service/definitions/{schema => }/user_batch.yaml (100%) delete mode 100644 specification/modules/application_services.rst create mode 100644 specification/modules/third_party_networks.rst diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index b68cbe18..43a6023b 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -226,7 +226,7 @@ paths: 200: description: The protocol was found and metadata returned. schema: - $ref: definitions/schema/protocol_metadata.yaml + $ref: definitions/protocol_metadata.yaml 401: description: |- The homeserver has not supplied credentials to the application service. @@ -279,7 +279,7 @@ paths: 200: description: The Matrix User IDs found with the given parameters. schema: - $ref: definitions/schema/user_batch.yaml + $ref: definitions/user_batch.yaml 401: description: |- The homeserver has not supplied credentials to the application service. @@ -330,7 +330,7 @@ paths: 200: description: At least one portal room was found. schema: - $ref: definitions/schema/location_batch.yaml + $ref: definitions/location_batch.yaml 401: description: |- The homeserver has not supplied credentials to the application service. @@ -375,7 +375,7 @@ paths: description: |- All found third party locations. schema: - $ref: definitions/schema/location_batch.yaml + $ref: definitions/location_batch.yaml 401: description: |- The homeserver has not supplied credentials to the application service. @@ -419,7 +419,7 @@ paths: description: |- An array of third party users. schema: - $ref: definitions/schema/user_batch.yaml + $ref: definitions/user_batch.yaml 401: description: |- The homeserver has not supplied credentials to the application service. diff --git a/api/application-service/definitions/schema/location.yaml b/api/application-service/definitions/location.yaml similarity index 100% rename from api/application-service/definitions/schema/location.yaml rename to api/application-service/definitions/location.yaml diff --git a/api/application-service/definitions/schema/location_batch.yaml b/api/application-service/definitions/location_batch.yaml similarity index 100% rename from api/application-service/definitions/schema/location_batch.yaml rename to api/application-service/definitions/location_batch.yaml diff --git a/api/application-service/definitions/schema/protocol.yaml b/api/application-service/definitions/protocol.yaml similarity index 100% rename from api/application-service/definitions/schema/protocol.yaml rename to api/application-service/definitions/protocol.yaml diff --git a/api/application-service/definitions/schema/protocol_metadata.yaml b/api/application-service/definitions/protocol_metadata.yaml similarity index 100% rename from api/application-service/definitions/schema/protocol_metadata.yaml rename to api/application-service/definitions/protocol_metadata.yaml diff --git a/api/application-service/definitions/schema/user.yaml b/api/application-service/definitions/user.yaml similarity index 100% rename from api/application-service/definitions/schema/user.yaml rename to api/application-service/definitions/user.yaml diff --git a/api/application-service/definitions/schema/user_batch.yaml b/api/application-service/definitions/user_batch.yaml similarity index 100% rename from api/application-service/definitions/schema/user_batch.yaml rename to api/application-service/definitions/user_batch.yaml diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 2fcf790b..efb4310f 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -39,7 +39,7 @@ paths: 200: description: The protocols supported by the homeserver. schema: - $ref: ../application-service/definitions/schema/protocol_metadata.yaml + $ref: ../application-service/definitions/protocol_metadata.yaml "/thirdparty/protocol/{protocol}": get: summary: Retrieve metadata about a specific protocol that the homeserver supports. @@ -58,7 +58,7 @@ paths: 200: description: The protocol was found and metadata returned. schema: - $ref: ../application-service/definitions/schema/protocol.yaml + $ref: ../application-service/definitions/protocol.yaml 404: description: The protocol is unknown. examples: @@ -97,7 +97,7 @@ paths: 200: description: At least one portal room was found. schema: - $ref: ../application-service/definitions/schema/location_batch.yaml + $ref: ../application-service/definitions/location_batch.yaml 404: description: No portal rooms were found. examples: @@ -130,7 +130,7 @@ paths: 200: description: The Matrix User IDs found with the given parameters. schema: - $ref: ../application-service/definitions/schema/user_batch.yaml + $ref: ../application-service/definitions/user_batch.yaml 404: description: The Matrix User ID was not found examples: @@ -157,7 +157,7 @@ paths: description: |- All found third party locations. schema: - $ref: ../application-service/definitions/schema/location_batch.yaml + $ref: ../application-service/definitions/location_batch.yaml 404: description: The Matrix room alias was not found examples: @@ -183,7 +183,7 @@ paths: description: |- An array of third party users. schema: - $ref: ../application-service/definitions/schema/user_batch.yaml + $ref: ../application-service/definitions/user_batch.yaml 404: description: The Matrix User ID was not found examples: diff --git a/specification/feature_profiles.rst b/specification/feature_profiles.rst index 97d0de0e..c6b8ef4c 100644 --- a/specification/feature_profiles.rst +++ b/specification/feature_profiles.rst @@ -42,7 +42,7 @@ Summary `Server Side Search`_ Optional Optional Optional Optional Optional `Server Administration`_ Optional Optional Optional Optional Optional `Event Context`_ Optional Optional Optional Optional Optional - `Application Services`_ Optional Optional Optional Optional Optional + `Third Party Networks`_ Optional Optional Optional Optional Optional ===================================== ========== ========== ========== ========== ========== *Please see each module for more details on what clients need to implement.* @@ -58,7 +58,7 @@ Summary .. _Server Side Search: `module:search`_ .. _Server Administration: `module:admin`_ .. _Event Context: `module:event-context`_ -.. _Application Services: `module:application-services`_ +.. _Third Party Networks: `module:third-party-networks`_ Clients ------- diff --git a/specification/modules/application_services.rst b/specification/modules/application_services.rst deleted file mode 100644 index 52e35dc8..00000000 --- a/specification/modules/application_services.rst +++ /dev/null @@ -1,28 +0,0 @@ -Application Services -==================== - -.. _module:application-services: - -An application service is a separate program that interacts with a homeserver -and provides various bits of functionality that would otherwise not make -sense to include directly in the homeserver. This ranges from bots, which can -often be interacted with, to bridges, which allow Matrix users to communicate -with users on third party networks. The following describes endpoints that a -Matrix client can use to interact with an application service through the -facilitation of the homeserver. - -Third Party Lookups -------------------- - -Application services can provide access to third party networks via bridging. -This allows Matrix users to communicate with users on other communication -platforms, with messages ferried back and forth by the application service. A -single application service may bridge multiple third party networks, and many -individual locations within those networks. A single third party network -location may be bridged to multiple Matrix rooms. - -In order for a client to join one of these bridged rooms, or communicate -directly with a user on a third party network, the following endpoints can be -used. - -{{third_party_lookup_cs_http_api}} \ No newline at end of file diff --git a/specification/modules/third_party_networks.rst b/specification/modules/third_party_networks.rst new file mode 100644 index 00000000..cd4ce414 --- /dev/null +++ b/specification/modules/third_party_networks.rst @@ -0,0 +1,20 @@ +Third Party Networks +==================== + +.. _module:third-party-networks: + +Application services can provide access to third party networks via bridging. +This allows Matrix users to communicate with users on other communication +platforms, with messages ferried back and forth by the application service. A +single application service may bridge multiple third party networks, and many +individual locations within those networks. A single third party network +location may be bridged to multiple Matrix rooms. + +Third Party Lookups +------------------- + +A client may wish to provide a rich interface for joining third party +locations and connecting with third party users. Information necessary for +such an interface is provided by third party lookups. + +{{third_party_lookup_cs_http_api}} \ No newline at end of file diff --git a/specification/targets.yaml b/specification/targets.yaml index ba3e3299..b9718bc4 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -68,7 +68,7 @@ groups: # reusable blobs of files when prefixed with 'group:' - modules/ignore_users.rst - modules/stickers.rst - modules/report_content.rst - - modules/application_services.rst + - modules/third_party_networks.rst title_styles: ["=", "-", "~", "+", "^", "`", "@", ":"] From af7460088fa756eb2b3afff2949f5ee281c2e2e0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 10:07:09 -0600 Subject: [PATCH 187/308] Use six for url quoting --- scripts/templating/matrix_templates/units.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 760aeb6b..12db409f 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -30,9 +30,8 @@ import re import subprocess import sys import yaml -import requests.utils from functools import reduce -from six.moves.urllib.parse import urlencode +from six.moves.urllib.parse import urlencode, quote matrix_doc_dir=reduce(lambda acc,_: os.path.dirname(acc), range(1, 5), os.path.abspath(__file__)) @@ -525,7 +524,7 @@ class MatrixUnits(Units): if param_loc == "path": path_template = path_template.replace( - "{%s}" % param_name, requests.utils.requote_uri(example) + "{%s}" % param_name, quote(example) ) elif param_loc == "query": if type(example) == list: From 8a4ba8c5cac07011159855735e96948f33787e08 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 10:16:32 -0600 Subject: [PATCH 188/308] Use Popen rather than run() --- scripts/templating/matrix_templates/units.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 12db409f..cd563346 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -838,21 +838,21 @@ class MatrixUnits(Units): tc_lines = [] if os.path.isdir(tc_path): logger.info("Generating towncrier changelog for: %s" % name) - p = subprocess.run( + p = subprocess.Popen( ['towncrier', '--version', 'Unreleased Changes', '--name', name, '--draft'], cwd=tc_path, - check=False, # We'll manually check the exit code stderr=subprocess.PIPE, stdout=subprocess.PIPE, ) + stdout, stderr = p.communicate() if p.returncode != 0: # Something broke - dump as much information as we can logger.error("Towncrier exited with code %s" % p.returncode) - logger.error(p.stdout.decode('UTF-8')) - logger.error(p.stderr.decode('UTF-8')) + logger.error(stdout.decode('UTF-8')) + logger.error(stderr.decode('UTF-8')) raw_log = "" else: - raw_log = p.stdout.decode('UTF-8') + raw_log = stdout.decode('UTF-8') # This is a bit of a hack, but it does mean that the log at least gets *something* # to tell us it broke From 527a5a2fdccd72d38a7b7d3e621b31b9128a23c3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 10:18:46 -0600 Subject: [PATCH 189/308] Require sudo see also: https://github.com/travis-ci/travis-ci/issues/9875 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 27e5cf20..68fc7743 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: go go: - 1.8 -sudo: false +sudo: required # we only need a single git commit git: From 1829589d23f9c23cbcb7f3266408cb5db80f5517 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 12:16:11 -0600 Subject: [PATCH 190/308] Convert 3pn errors to new definition --- .../application_service.yaml | 42 +++++++++---------- api/client-server/third_party_lookup.yaml | 10 ++--- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index 43a6023b..b459e29f 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -131,7 +131,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -140,7 +140,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 404: description: |- The application service indicates that this room alias does not exist. @@ -150,7 +150,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml "/users/{userId}": get: summary: Query if a user should exist on the application service. @@ -187,7 +187,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -196,7 +196,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 404: description: |- The application service indicates that this user does not exist. @@ -206,7 +206,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml "/_matrix/app/unstable/thirdparty/protocol/{protocol}": get: summary: Retrieve metadata about a specific protocol that the application service supports. @@ -236,7 +236,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -245,7 +245,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 404: description: No protocol was found with the given path. examples: @@ -253,7 +253,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml "/_matrix/app/unstable/thirdparty/user/{protocol}": get: summary: Retrieve the Matrix User ID of a corresponding third party user. @@ -289,7 +289,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -298,7 +298,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 404: description: No users were found with the given parameters. examples: @@ -306,7 +306,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml "/_matrix/app/unstable/thirdparty/location/{protocol}": get: summary: Retreive Matrix-side portal rooms leading to a third party location. @@ -340,7 +340,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -349,7 +349,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 404: description: No mappings were found with the given parameters. examples: @@ -357,7 +357,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml "/_matrix/app/unstable/thirdparty/location": get: summary: Reverse-lookup third party locations given a Matrix room alias. @@ -385,7 +385,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -394,7 +394,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_FORBIDDEN" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 404: description: No mappings were found with the given parameters. examples: @@ -402,7 +402,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml "/_matrix/app/unstable/thirdparty/user": get: summary: Reverse-lookup third party users given a Matrix User ID. @@ -429,7 +429,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 403: description: |- The credentials supplied by the homeserver were rejected. @@ -438,7 +438,7 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_UNAUTHORIZED" } schema: - $ref: ../client-server/definitions/error.yaml + $ref: ../client-server/definitions/errors/error.yaml 404: description: No mappings were found with the given parameters. examples: @@ -446,4 +446,4 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - $ref: ../client-server/definitions/error.yaml \ No newline at end of file + $ref: ../client-server/definitions/errors/error.yaml \ No newline at end of file diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index efb4310f..55d4c70a 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -66,7 +66,7 @@ paths: "errcode": "M_NOT_FOUND" } schema: - $ref: definitions/error.yaml + $ref: definitions/errors/error.yaml "/thirdparty/location/{protocol}": get: summary: Retreive Matrix-side portals rooms leading to a third party location. @@ -105,7 +105,7 @@ paths: "errcode": "M_NOT_FOUND" } schema: - $ref: definitions/error.yaml + $ref: definitions/errors/error.yaml "/thirdparty/user/{protocol}": get: summary: Retrieve the Matrix User ID of a corresponding third party user. @@ -138,7 +138,7 @@ paths: "errcode": "M_NOT_FOUND" } schema: - $ref: definitions/error.yaml + $ref: definitions/errors/error.yaml "/thirdparty/location": get: summary: Reverse-lookup third party locations given a Matrix room alias. @@ -165,7 +165,7 @@ paths: "errcode": "M_NOT_FOUND" } schema: - $ref: definitions/error.yaml + $ref: definitions/errors/error.yaml "/thirdparty/user": get: summary: Reverse-lookup third party users given a Matrix User ID. @@ -191,4 +191,4 @@ paths: "errcode": "M_NOT_FOUND" } schema: - $ref: definitions/error.yaml \ No newline at end of file + $ref: definitions/errors/error.yaml \ No newline at end of file From 1e500fd00563d52006807886cb223be107482d27 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 13:18:27 -0600 Subject: [PATCH 191/308] Add newsfragment for third party network endpoints Specifying multiple entries for the same PR would be covered under: https://github.com/hawkowl/towncrier/pull/119 --- changelogs/client_server/newsfragments/1353.new | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1353.new diff --git a/changelogs/client_server/newsfragments/1353.new b/changelogs/client_server/newsfragments/1353.new new file mode 100644 index 00000000..0af0c520 --- /dev/null +++ b/changelogs/client_server/newsfragments/1353.new @@ -0,0 +1 @@ +``GET /thirdparty/*`` Endpoints From 51faf8ed2e4a63d4cfd6d23183698ed169956cc0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 13:46:42 -0600 Subject: [PATCH 192/308] Convert the general query API to swagger Includes "moving" the directory query API into the same area. This will make more sense once the other query types are specified. --- api/server-server/directory.yaml | 3 +- api/server-server/query_general.yaml | 44 ++++++++++++++++++++++++++++ specification/server_server_api.rst | 11 +------ 3 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 api/server-server/query_general.yaml diff --git a/api/server-server/directory.yaml b/api/server-server/directory.yaml index 516b1bf8..94f115b9 100644 --- a/api/server-server/directory.yaml +++ b/api/server-server/directory.yaml @@ -1,4 +1,5 @@ # Copyright 2017 Kamax.io +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +15,7 @@ swagger: '2.0' info: - title: "Matrix Federation Query Directory API" + title: "Matrix Federation Query API" version: "1.0.0" host: localhost:8448 schemes: diff --git a/api/server-server/query_general.yaml b/api/server-server/query_general.yaml new file mode 100644 index 00000000..97e3406e --- /dev/null +++ b/api/server-server/query_general.yaml @@ -0,0 +1,44 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Query API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/query/{queryType}": + get: + summary: Query for information + description: |- + Performs a single query request on the receiving homeserver. The query string + arguments are dependent on which type of query is being made. Known query types + are specified as their own endpoints as an extension to this definition. + operationId: queryInfo + parameters: + - in: path + name: queryType + type: string + description: The type of query to make + required: true + x-example: profile + responses: + 200: + description: |- + The query response. The schema varies depending on the query being made. diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f2b3204a..66c06c8b 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -801,16 +801,7 @@ Retrieves all of the transactions later than any version given by the "v" arguments. -To make a query:: - - GET .../query/ - Query args: as specified by the individual query types - Response: JSON encoding of a response object - -Performs a single query request on the receiving homeserver. The Query Type -part of the path specifies the kind of query being made, and its query -arguments have a meaning specific to that kind of query. The response is a -JSON-encoded object whose meaning also depends on the kind of query. +{{query_general_ss_http_api}} To join a room:: From e5da1e17412c99437d2af50b9b62c150c1c40662 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 21:34:52 -0600 Subject: [PATCH 193/308] Improve upon the transaction schema and examples Reduces data duplication and makes the objects more reusable for other areas of the spec --- .../definitions/full_transaction.yaml | 28 --- api/server-server/definitions/pdu.yaml | 160 ++++-------------- .../definitions/unsigned_pdu.yaml | 119 +++++++++++++ .../examples/full_transaction.json | 6 - api/server-server/examples/pdu.json | 16 +- api/server-server/examples/unsigned_pdu.json | 17 ++ api/server-server/transactions.yaml | 15 +- scripts/templating/matrix_templates/units.py | 13 +- 8 files changed, 196 insertions(+), 178 deletions(-) delete mode 100644 api/server-server/definitions/full_transaction.yaml create mode 100644 api/server-server/definitions/unsigned_pdu.yaml delete mode 100644 api/server-server/examples/full_transaction.json create mode 100644 api/server-server/examples/unsigned_pdu.json diff --git a/api/server-server/definitions/full_transaction.yaml b/api/server-server/definitions/full_transaction.yaml deleted file mode 100644 index 39250433..00000000 --- a/api/server-server/definitions/full_transaction.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2018 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -type: object -title: Transaction -description: Transaction -example: - $ref: "../examples/full_transaction.json" -allOf: - - $ref: "transaction.yaml" - - type: object - properties: - edus: - type: array - description: List of ephemeral messages. May be omitted if there are no ephemeral messages to be sent. - items: - $ref: "edu.yaml" -required: ['origin', 'origin_server_ts', 'pdus'] \ No newline at end of file diff --git a/api/server-server/definitions/pdu.yaml b/api/server-server/definitions/pdu.yaml index b3f25ccf..2b6328be 100644 --- a/api/server-server/definitions/pdu.yaml +++ b/api/server-server/definitions/pdu.yaml @@ -16,131 +16,37 @@ title: Persistent Data Unit description: A persistent data unit (event) example: $ref: "../examples/pdu.json" -properties: - room_id: - type: string - description: Room identifier - example: "!abc123:matrix.org" - sender: - type: string - description: The ID of the user sending the event - example: "@someone:matrix.org" - origin: - type: string - description: The ``server_name`` of the homeserver that created this event - example: "matrix.org" - origin_server_ts: - type: integer - format: int64 - description: Timestamp in milliseconds on origin homeserver when this event was created. - example: 1234567890 - type: - type: string - description: Event type - required: true - example: "m.room.message" - state_key: - type: string - description: |- - If this key is present, the event is a state event, and it will replace previous events - with the same ``type`` and ``state_key`` in the room state. - example: "my_key" - content: - type: object - description: The content of the event - prev_events: - type: array - description: |- - Event IDs and hashes of the most recent events in the room that the homeserver was aware - of when it made this event - items: - type: array - maxItems: 2 - minItems: 2 - items: - - type: string - title: Event ID - example: "$abc123:matrix.org" - - type: object - title: Event Hash - example: { - "sha256": "abase64encodedsha256hashshouldbe43byteslong" - } - properties: - sha256: - type: string - description: The event hash - example: abase64encodedsha256hashshouldbe43byteslong - required: ['sha256'] - depth: - type: integer - description: The maximum depth of the ``prev_events``, plus one - example: 12 - auth_events: - type: array - description: Event IDs and hashes for the "auth events" of this event - items: - type: array - maxItems: 2 - minItems: 2 - items: - - type: string - title: Event ID - example: "$abc123:matrix.org" - - type: object - title: Event Hash - example: { - "sha256": "abase64encodedsha256hashshouldbe43byteslong" - } - properties: - sha256: - type: string - description: The event hash - example: abase64encodedsha256hashshouldbe43byteslong - required: ['sha256'] - hashes: - type: object - title: Event Hash - description: Hashes of the PDU, following the algorithm specified in `Signing Events`_ - example: { - "sha256": "thishashcoversallfieldsincasethisisredacted" - } +allOf: + - $ref: "unsigned_pdu.yaml" + - type: object properties: - sha256: - type: string - description: The hash - example: thishashcoversallfieldsincasethisisredacted - required: ['sha256'] - signatures: - type: object - description: |- - Signatures of the redacted PDU, following the algorithm specified in `Signing Events`_ - example: { - "example.com": { - "ed25519:key_version:": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus" - } - } - additionalProperties: - type: object - title: Server Signatures - additionalProperties: - type: string - redacts: - type: string - description: For redaction events, the ID of the event being redacted - example: "$def456:matrix.org" - unsigned: - type: object - description: Additional data added by the origin server but not covered by the ``signatures`` -required: - - room_id - - sender - - origin - - origin_server_ts - - type - - content - - prev_events - - depth - - auth_events - - hashes - - signatures + hashes: + type: object + title: Event Hash + description: Hashes of the PDU, following the algorithm specified in `Signing Events`_ + example: { + "sha256": "thishashcoversallfieldsincasethisisredacted" + } + properties: + sha256: + type: string + description: The hash + example: thishashcoversallfieldsincasethisisredacted + required: ['sha256'] + signatures: + type: object + description: |- + Signatures of the redacted PDU, following the algorithm specified in `Signing Events`_ + example: { + "example.com": { + "ed25519:key_version:": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus" + } + } + additionalProperties: + type: object + title: Server Signatures + additionalProperties: + type: string + required: + - hashes + - signatures diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml new file mode 100644 index 00000000..173bddd4 --- /dev/null +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -0,0 +1,119 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +title: Unsigned Persistent Data Unit +description: An unsigned persistent data unit (event) +example: + $ref: "../examples/unsigned_pdu.json" +properties: + room_id: + type: string + description: Room identifier + example: "!abc123:matrix.org" + sender: + type: string + description: The ID of the user sending the event + example: "@someone:matrix.org" + origin: + type: string + description: The ``server_name`` of the homeserver that created this event + example: "matrix.org" + origin_server_ts: + type: integer + format: int64 + description: Timestamp in milliseconds on origin homeserver when this event was created. + example: 1234567890 + type: + type: string + description: Event type + required: true + example: "m.room.message" + state_key: + type: string + description: |- + If this key is present, the event is a state event, and it will replace previous events + with the same ``type`` and ``state_key`` in the room state. + example: "my_key" + content: + type: object + description: The content of the event + example: {"key": "value"} + prev_events: + type: array + description: |- + Event IDs and hashes of the most recent events in the room that the homeserver was aware + of when it made this event + items: + type: array + maxItems: 2 + minItems: 2 + items: + - type: string + title: Event ID + example: "$abc123:matrix.org" + - type: object + title: Event Hash + example: { + "sha256": "abase64encodedsha256hashshouldbe43byteslong" + } + properties: + sha256: + type: string + description: The event hash + example: abase64encodedsha256hashshouldbe43byteslong + required: ['sha256'] + depth: + type: integer + description: The maximum depth of the ``prev_events``, plus one + example: 12 + auth_events: + type: array + description: Event IDs and hashes for the "auth events" of this event + items: + type: array + maxItems: 2 + minItems: 2 + items: + - type: string + title: Event ID + example: "$abc123:matrix.org" + - type: object + title: Event Hash + example: { + "sha256": "abase64encodedsha256hashshouldbe43byteslong" + } + properties: + sha256: + type: string + description: The event hash + example: abase64encodedsha256hashshouldbe43byteslong + required: ['sha256'] + redacts: + type: string + description: For redaction events, the ID of the event being redacted + example: "$def456:matrix.org" + unsigned: + type: object + description: Additional data added by the origin server but not covered by the ``signatures`` + example: {"key": "value"} +required: + - room_id + - sender + - origin + - origin_server_ts + - type + - content + - prev_events + - depth + - auth_events diff --git a/api/server-server/examples/full_transaction.json b/api/server-server/examples/full_transaction.json deleted file mode 100644 index c453d6ed..00000000 --- a/api/server-server/examples/full_transaction.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "origin": "matrix.org", - "origin_server_ts": 1234567890, - "pdus": [{"$ref": "pdu.json"}], - "edus": [{"$ref": "edu.json"}] -} \ No newline at end of file diff --git a/api/server-server/examples/pdu.json b/api/server-server/examples/pdu.json index 531f4a37..81981b23 100644 --- a/api/server-server/examples/pdu.json +++ b/api/server-server/examples/pdu.json @@ -1,16 +1,5 @@ { - "room_id": "!UcYsUzyxTGDxLBEvLy:example.org", - "sender": "@alice:example.com", - "origin": "example.com", - "event_id": "$a4ecee13e2accdadf56c1025:example.com", - "origin_server_ts": 1404838188000, - "type": "m.room.message", - "prev_events": [ - [ - "$af232176:example.org", - {"sha256": "abase64encodedsha256hashshouldbe43byteslong"} - ] - ], + "$ref": "unsigned_pdu.json", "hashes": { "sha256": "thishashcoversallfieldsincasethisisredacted" }, @@ -18,8 +7,5 @@ "example.com": { "ed25519:key_version:": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus" } - }, - "content": { - "key": "value" } } \ No newline at end of file diff --git a/api/server-server/examples/unsigned_pdu.json b/api/server-server/examples/unsigned_pdu.json new file mode 100644 index 00000000..6ed4b79d --- /dev/null +++ b/api/server-server/examples/unsigned_pdu.json @@ -0,0 +1,17 @@ +{ + "room_id": "!UcYsUzyxTGDxLBEvLy:example.org", + "sender": "@alice:example.com", + "origin": "example.com", + "event_id": "$a4ecee13e2accdadf56c1025:example.com", + "origin_server_ts": 1404838188000, + "type": "m.room.message", + "prev_events": [ + [ + "$af232176:example.org", + {"sha256": "abase64encodedsha256hashshouldbe43byteslong"} + ] + ], + "content": { + "key": "value" + } +} \ No newline at end of file diff --git a/api/server-server/transactions.yaml b/api/server-server/transactions.yaml index 389eecc7..13ba6826 100644 --- a/api/server-server/transactions.yaml +++ b/api/server-server/transactions.yaml @@ -43,8 +43,21 @@ paths: - in: body name: body type: object + required: true schema: - $ref: "definitions/full_transaction.yaml" + allOf: + - $ref: "definitions/transaction.yaml" + - type: object + properties: + edus: + type: array + description: List of ephemeral messages. May be omitted if there are no ephemeral messages to be sent. + items: + $ref: "definitions/edu.yaml" + example: { + "$ref": "examples/transaction.json", + "edus": [{"$ref": "edu.json"}] # Relative to the examples directory + } responses: 200: # TODO: Spec this (and figure out what it is) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 7819dbf9..8037bb61 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -138,6 +138,10 @@ def inherit_parents(obj): Recurse through the 'allOf' declarations in the object """ logger.debug("inherit_parents %r" % obj) + + if isinstance(obj, list): + return [inherit_parents(obj[i]) for i in range(0, len(obj))] + parents = obj.get("allOf", []) if not parents: return obj @@ -379,6 +383,10 @@ def get_tables_for_response(schema): def get_example_for_schema(schema): """Returns a python object representing a suitable example for this object""" schema = inherit_parents(schema) + + if isinstance(schema, list): + return [get_example_for_schema(v) for v in schema] + if 'example' in schema: example = schema['example'] return example @@ -398,7 +406,10 @@ def get_example_for_schema(schema): if proptype == 'array': if 'items' not in schema: raise Exception('"array" property has neither items nor example') - return [get_example_for_schema(schema['items'])] + result = get_example_for_schema(schema['items']) + if isinstance(result, list): + return result + return [result] if proptype == 'integer': return 0 From baf19cc741071bf6e5b5c3235f56a9cd99488ae8 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 13 Jul 2018 10:05:14 -0600 Subject: [PATCH 194/308] Add newsfragment for 1379 --- changelogs/client_server/newsfragments/1379.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1379.clarification diff --git a/changelogs/client_server/newsfragments/1379.clarification b/changelogs/client_server/newsfragments/1379.clarification new file mode 100644 index 00000000..122b3900 --- /dev/null +++ b/changelogs/client_server/newsfragments/1379.clarification @@ -0,0 +1 @@ +Document guest access in ``/createRoom`` presets From cf93c88337c15b491f9ae069d5f08ffd69c9fdc2 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 13 Jul 2018 15:44:33 -0600 Subject: [PATCH 195/308] Convert third party invite exchange to swagger --- api/server-server/third_party_invite.yaml | 190 ++++++++++++++++++++++ specification/server_server_api.rst | 33 +--- 2 files changed, 192 insertions(+), 31 deletions(-) create mode 100644 api/server-server/third_party_invite.yaml diff --git a/api/server-server/third_party_invite.yaml b/api/server-server/third_party_invite.yaml new file mode 100644 index 00000000..30d8fe9d --- /dev/null +++ b/api/server-server/third_party_invite.yaml @@ -0,0 +1,190 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Third Party Invites API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/exchange_third_party_invite/{roomId}": + put: + summary: Request a server to auth a third party invite event + description: |- + The receiving server will verify the partial ``m.room.member`` event + given in the request body. If valid, the receiving server will issue + an invite as per the `Inviting Users`_ section before returning a + response to this request. + operationId: exchangeThirdPartyInvite + parameters: + - in: path + name: roomId + type: string + description: The room ID to exchange a third party invite in + required: true + x-example: "!abc123:matrix.org" + - in: body + name: body + type: object + description: A partial ``m.room.member`` event + required: true + schema: + type: object + properties: + type: + type: string + description: The event type. Must be ``m.room.member`` + example: "m.room.member" + room_id: + type: string + description: |- + The room ID the event is for. Must match the ID given in + the path. + example: "!abc123:matrix.org" + sender: + type: string + description: |- + The user ID of the user who sent the original ``m.room.third_party_invite`` + event. + example: "@joe:matrix.org" + state_key: + type: string + description: The user ID of the invited user + example: "@someone:example.org" + content: + type: object + description: The event content + title: Event Content + properties: + membership: + type: string + description: The membership state. Must be ``invite`` + example: invite + third_party_invite: + type: object + description: The third party invite + properties: + display_name: + type: string + description: |- + A name which can be displayed to represent the user instead of their + third party identifier. + example: "alice" + signed: + type: object + description: |- + A block of content which has been signed, which servers can use to + verify the event. + properties: + signatures: + type: object + description: The server signatures for this event. + additionalProperties: + type: object + title: Server Signatures + additionalProperties: + type: string + example: { + "magic.forest": { + "ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg" + } + } + mxid: + type: string + description: The invited matrix user ID + example: "@alice:localhost" + token: + type: string + description: The token used to verify the event + example: abc123 + required: ['signatures', 'mxid', 'token'] + example: { + "mxid": "@alice:localhost", + "token": "abc123", + "signatures": { + "magic.forest": { + "ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg" + } + } + } + required: ['display_name', 'signed'] + example: { + "display_name": "alice", + "signed": { + "mxid": "@alice:localhost", + "token": "abc123", + "signatures": { + "magic.forest": { + "ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg" + } + } + } + } + required: ['membership', 'third_party_invite'] + example: { + "membership": "invite", + "third_party_invite": { + "display_name": "alice", + "signed": { + "mxid": "@alice:localhost", + "token": "abc123", + "signatures": { + "magic.forest": { + "ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg" + } + } + } + } + } + required: + - type + - room_id + - sender + - state_key + - content + example: { + "type": "m.room.member", + "room_id": "!abc123:matrix.org", + "sender": "@joe:matrix.org", + "state_key": "@someone:example.org", + "content": { + "membership": "invite", + "third_party_invite": { + "display_name": "alice", + "signed": { + "mxid": "@alice:localhost", + "token": "abc123", + "signatures": { + "magic.forest": { + "ed25519:3": "fQpGIW1Snz+pwLZu6sTy2aHy/DYWWTspTJRPyNp0PKkymfIsNffysMl6ObMMFdIJhk6g6pwlIqZ54rxo8SLmAg" + } + } + } + } + } + } + responses: + 200: + description: The invite has been issued successfully. + examples: + application/json: {} + schema: + type: object + description: An empty object + example: {} diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 66c06c8b..43524aca 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -1104,38 +1104,9 @@ If the invited homeserver is in the room the invite came from, it can auth the event and send it. However, if the invited homeserver isn't in the room the invite came from, it -will need to request the room's homeserver to auth the event:: +will need to request the room's homeserver to auth the event. - PUT .../exchange_third_party_invite/{roomId} - -Where ``roomId`` is the ID of the room the invite is for. - -The required fields in the JSON body are: - -==================== ======= ================================================== - Key Type Description -==================== ======= ================================================== -``type`` String The event type. Must be ``m.room.member``. -``room_id`` String The ID of the room the event is for. Must be the - same as the ID specified in the path. -``sender`` String The Matrix ID of the user who sent the original - ``m.room.third_party_invite``. -``state_key`` String The Matrix ID of the invited user. -``content`` Object The content of the event. -==================== ======= ================================================== - -Where the ``content`` key contains the content for the ``m.room.member`` event -as described in the `Client-Server API`_. Its ``membership`` key must be -``invite`` and its content must include the ``third_party_invite`` object. - -The inviting homeserver will then be able to authenticate the event. It will send -a fully authenticated event to the invited homeserver as described in the `Inviting -to a room`_ section above. - -Once the invited homeserver responds with the event to which it appended its -signature, the inviting homeserver will respond with ``200 OK`` and an empty body -(``{}``) to the initial request on ``/exchange_third_party_invite/{roomId}`` and -send the now verified ``m.room.member`` invite event to the room's members. +{{third_party_invite_ss_http_api}} Verifying the invite ++++++++++++++++++++ From 272f3d843245eefe6b0874193af0e848e613a7ea Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 13 Jul 2018 15:49:49 -0600 Subject: [PATCH 196/308] Fix documentation link --- api/server-server/third_party_invite.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server-server/third_party_invite.yaml b/api/server-server/third_party_invite.yaml index 30d8fe9d..42bdae23 100644 --- a/api/server-server/third_party_invite.yaml +++ b/api/server-server/third_party_invite.yaml @@ -29,7 +29,7 @@ paths: description: |- The receiving server will verify the partial ``m.room.member`` event given in the request body. If valid, the receiving server will issue - an invite as per the `Inviting Users`_ section before returning a + an invite as per the `Inviting to a room`_ section before returning a response to this request. operationId: exchangeThirdPartyInvite parameters: From cfac4a1ab38ed05fc677c999b8ab379e1ff97eca Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 17 Jul 2018 08:48:31 -0600 Subject: [PATCH 197/308] Use towncrier 18.6.0 final --- scripts/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 1f92c3dc..2a7d7ff8 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -7,5 +7,5 @@ Jinja2 >= 2.9.6 jsonschema >= 2.6.0 PyYAML >= 3.12 requests >= 2.18.4 -towncrier == 18.6.0rc1 +towncrier == 18.6.0 six >= 1.11.0 \ No newline at end of file From 98c607bff5e59d9d1db4a5af34fb197ac996d428 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 17 Jul 2018 10:27:01 -0600 Subject: [PATCH 198/308] Advertise python 3.4 support and use that in Travis CI We also don't need to update virtualenv, or use python3.5 specific packages. --- .travis.yml | 5 ++--- README.rst | 4 ++-- scripts/test-and-build.sh | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 68fc7743..9e9363af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,8 @@ git: depth: 1 install: -- sudo apt-get update -- sudo apt-get install python3.5 python3.5-dev -- sudo pip install --upgrade virtualenv + - sudo apt-get update + - sudo apt-get install python3 python3-dev script: - ./scripts/test-and-build.sh diff --git a/README.rst b/README.rst index c0949d9e..b8847bfb 100644 --- a/README.rst +++ b/README.rst @@ -41,9 +41,9 @@ specs and event schemas in this repository. Preparation ----------- -To use the scripts, it is best to create a Python 3.5+ virtualenv as follows:: +To use the scripts, it is best to create a Python 3.4+ virtualenv as follows:: - virtualenv env + virtualenv -p python3 env env/bin/pip install -r scripts/requirements.txt (Benjamin Synders has contributed a script for `Nix`_ users, which can be diff --git a/scripts/test-and-build.sh b/scripts/test-and-build.sh index 397b620e..710b03dd 100755 --- a/scripts/test-and-build.sh +++ b/scripts/test-and-build.sh @@ -4,7 +4,7 @@ set -ex cd `dirname $0`/.. -virtualenv -p python3.5 env +virtualenv -p python3 env . env/bin/activate # Print out the python versions for debugging purposes From 9e6f2b2fd01a3f1ecd7d9e56169c2d52690b0f49 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 18 Jul 2018 08:29:13 -0600 Subject: [PATCH 199/308] Add newsfragment --- changelogs/client_server/newsfragments/1373.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1373.clarification diff --git a/changelogs/client_server/newsfragments/1373.clarification b/changelogs/client_server/newsfragments/1373.clarification new file mode 100644 index 00000000..21e18966 --- /dev/null +++ b/changelogs/client_server/newsfragments/1373.clarification @@ -0,0 +1 @@ +Describe the rate limit error response schema From bafdcf364011c5e8e1337b23762eea71f8256aab Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 18 Jul 2018 10:37:56 -0600 Subject: [PATCH 200/308] Full stops, spelling, and operation IDs. --- api/server-server/definitions/keys.yaml | 18 ++++++------ .../definitions/keys_query_response.yaml | 2 +- api/server-server/keys_query.yaml | 28 +++++++++---------- api/server-server/keys_server.yaml | 4 +-- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/api/server-server/definitions/keys.yaml b/api/server-server/definitions/keys.yaml index 5d1c39de..b08f2465 100644 --- a/api/server-server/definitions/keys.yaml +++ b/api/server-server/definitions/keys.yaml @@ -19,12 +19,12 @@ example: properties: server_name: type: string - description: DNS name of the homeserver + description: DNS name of the homeserver. required: true # TODO: Verify example: "example.org" verify_keys: type: object - description: Public keys of the homeserver for verifying digital signatures + description: Public keys of the homeserver for verifying digital signatures. required: true # TODO: Verify additionalProperties: type: object @@ -42,7 +42,7 @@ properties: example: "Base+64+Encoded+Signature+Verification+Key" old_verify_keys: type: object - description: The public keys that the server used to use and when it stopped using them + description: The public keys that the server used to use and when it stopped using them. additionalProperties: type: object title: Old Verify Key @@ -56,17 +56,17 @@ properties: expired_ts: type: integer format: int64 - description: The expiration time + description: The expiration time. required: true example: 922834800000 key: type: string - description: The key + description: The key. required: true example: "Base+64+Encoded+Signature+Verification+Key" signatures: type: object - description: Digital signatures for this object signed using the ``verify_keys`` + description: Digital signatures for this object signed using the ``verify_keys``. additionalProperties: type: object title: Signed Server @@ -80,17 +80,17 @@ properties: name: Encoded Signature Verification Key tls_fingerprints: type: array - description: Hashes of X.509 TLS certificates used by this server encoded as `Unpadded Base64`_ + description: Hashes of X.509 TLS certificates used by this server encoded as `Unpadded Base64`_. items: type: object title: TLS Fingerprint properties: sha256: type: string - description: The encoded fingerprint + description: The encoded fingerprint. example: Base+64+Encoded+SHA-256-Fingerprint valid_until_ts: type: integer format: int64 - description: POSIX timestamp when the list of valid keys should be refreshed + description: POSIX timestamp when the list of valid keys should be refreshed. example: 1052262000000 diff --git a/api/server-server/definitions/keys_query_response.yaml b/api/server-server/definitions/keys_query_response.yaml index 8223f9ec..bf2d238f 100644 --- a/api/server-server/definitions/keys_query_response.yaml +++ b/api/server-server/definitions/keys_query_response.yaml @@ -22,6 +22,6 @@ properties: server_keys: type: array title: Server Keys - description: The server keys + description: The server keys. items: $ref: "keys.yaml" diff --git a/api/server-server/keys_query.yaml b/api/server-server/keys_query.yaml index 1ba0f5ba..f41cb35b 100644 --- a/api/server-server/keys_query.yaml +++ b/api/server-server/keys_query.yaml @@ -25,27 +25,27 @@ produces: paths: "/query/{serverName}/{keyId}": get: - summary: Retreive a server key - description: Retreive a server key - operationId: getQueryKeys + summary: Retrieve a server key. + description: Retrieve a server key. + operationId: perspectivesKeyQuery parameters: - in: path name: serverName type: string - description: Server name + description: Server name. required: true x-example: matrix.org - in: path name: keyId type: string - description: Key ID + description: Key ID. required: true x-example: TODO # No examples in spec so far - in: query name: minimum_valid_until_ts type: integer format: int64 - description: Minimum Valid Until Milliseconds + description: Minimum Valid Until Milliseconds. required: true # TODO: Verify x-example: 1234567890 responses: @@ -55,9 +55,9 @@ paths: $ref: "definitions/keys_query_response.yaml" "/query": post: - summary: Retreive a server key - description: Retreive a server key - operationId: postQueryKeys + summary: Retrieve a server key + description: Retrieve a server key. + operationId: bulkPerspectivesKeyQuery parameters: - in: body name: body @@ -76,24 +76,24 @@ paths: properties: server_keys: type: object - description: The query criteria + description: The query criteria. additionalProperties: type: object name: ServerName - description: The server names to query + description: The server names to query. additionalProperties: type: object title: Query Criteria - description: The server keys to query + description: The server keys to query. properties: minimum_valid_until_ts: type: integer format: int64 - description: Minimum Valid Until MS + description: Minimum Valid Until MS. example: 1234567890 required: ['server_keys'] responses: 200: - description: The keys for the server + description: The keys for the server. schema: $ref: "definitions/keys_query_response.yaml" diff --git a/api/server-server/keys_server.yaml b/api/server-server/keys_server.yaml index 819a2599..46beeebb 100644 --- a/api/server-server/keys_server.yaml +++ b/api/server-server/keys_server.yaml @@ -26,7 +26,7 @@ paths: "/server/{keyId}": get: summary: Get the server's key - description: Get the server's key + description: Get the server's key. operationId: getServerKey parameters: - in: path @@ -37,6 +37,6 @@ paths: x-example: TODO # No examples in the spec so far responses: 200: - description: The server's keys + description: The server's keys. schema: $ref: "definitions/keys.yaml" From 89384c96c18310d084fa3c2695c51109780b9f53 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 18 Jul 2018 11:06:23 -0600 Subject: [PATCH 201/308] Full stops, language. --- api/server-server/definitions/edu.yaml | 10 ++++----- api/server-server/definitions/pdu.yaml | 8 +++---- .../definitions/transaction.yaml | 6 ++--- .../definitions/unsigned_pdu.yaml | 22 +++++++++---------- api/server-server/events.yaml | 21 +++++++++--------- 5 files changed, 34 insertions(+), 33 deletions(-) diff --git a/api/server-server/definitions/edu.yaml b/api/server-server/definitions/edu.yaml index 4597ca62..af12752c 100644 --- a/api/server-server/definitions/edu.yaml +++ b/api/server-server/definitions/edu.yaml @@ -22,16 +22,16 @@ example: properties: edu_type: type: string - description: The type of ephemeral message - example: "!abc123:matrix.org" + description: The type of ephemeral message. + example: "m.presence" origin: type: string - description: The server name sending the ephemeral message + description: The server name sending the ephemeral message. example: "matrix.org" destination: type: string - description: The server name receiving the ephemeral message + description: The server name receiving the ephemeral message. example: "elsewhere.com" content: type: object - description: The content of the ephemeral message + description: The content of the ephemeral message. diff --git a/api/server-server/definitions/pdu.yaml b/api/server-server/definitions/pdu.yaml index 2b6328be..a5da57c0 100644 --- a/api/server-server/definitions/pdu.yaml +++ b/api/server-server/definitions/pdu.yaml @@ -23,20 +23,20 @@ allOf: hashes: type: object title: Event Hash - description: Hashes of the PDU, following the algorithm specified in `Signing Events`_ + description: Hashes of the PDU, following the algorithm specified in `Signing Events`_. example: { "sha256": "thishashcoversallfieldsincasethisisredacted" } properties: sha256: type: string - description: The hash - example: thishashcoversallfieldsincasethisisredacted + description: The hash. + example: thishashcoversallfieldsincasthisisredacted required: ['sha256'] signatures: type: object description: |- - Signatures of the redacted PDU, following the algorithm specified in `Signing Events`_ + Signatures of the redacted PDU, following the algorithm specified in `Signing Events`_. example: { "example.com": { "ed25519:key_version:": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus" diff --git a/api/server-server/definitions/transaction.yaml b/api/server-server/definitions/transaction.yaml index 630d2ba3..e4c1f559 100644 --- a/api/server-server/definitions/transaction.yaml +++ b/api/server-server/definitions/transaction.yaml @@ -20,16 +20,16 @@ properties: origin: type: string description: |- - The ``server_name`` of hoemserver sending this transaction + The ``server_name`` of the hoemserver sending this transaction. example: "example.org" origin_server_ts: type: integer format: int64 - description: Timestamp in milliseconds on originating homeserver when this transaction started + description: Timestamp in milliseconds on originating homeserver when this transaction started. example: 1234567890 pdus: type: array - description: List of persistent updates to rooms + description: List of persistent updates to rooms. items: $ref: "pdu.yaml" required: ['origin', 'origin_server_ts', 'pdus'] \ No newline at end of file diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml index 173bddd4..ca0a1183 100644 --- a/api/server-server/definitions/unsigned_pdu.yaml +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -19,15 +19,15 @@ example: properties: room_id: type: string - description: Room identifier + description: Room identifier. example: "!abc123:matrix.org" sender: type: string - description: The ID of the user sending the event + description: The ID of the user sending the event. example: "@someone:matrix.org" origin: type: string - description: The ``server_name`` of the homeserver that created this event + description: The ``server_name`` of the homeserver that created this event. example: "matrix.org" origin_server_ts: type: integer @@ -47,13 +47,13 @@ properties: example: "my_key" content: type: object - description: The content of the event + description: The content of the event. example: {"key": "value"} prev_events: type: array description: |- Event IDs and hashes of the most recent events in the room that the homeserver was aware - of when it made this event + of when it made this event. items: type: array maxItems: 2 @@ -70,16 +70,16 @@ properties: properties: sha256: type: string - description: The event hash + description: The event hash. example: abase64encodedsha256hashshouldbe43byteslong required: ['sha256'] depth: type: integer - description: The maximum depth of the ``prev_events``, plus one + description: The maximum depth of the ``prev_events``, plus one. example: 12 auth_events: type: array - description: Event IDs and hashes for the "auth events" of this event + description: Event IDs and hashes for the "auth events" of this event. items: type: array maxItems: 2 @@ -96,16 +96,16 @@ properties: properties: sha256: type: string - description: The event hash + description: The event hash. example: abase64encodedsha256hashshouldbe43byteslong required: ['sha256'] redacts: type: string - description: For redaction events, the ID of the event being redacted + description: For redaction events, the ID of the event being redacted. example: "$def456:matrix.org" unsigned: type: object - description: Additional data added by the origin server but not covered by the ``signatures`` + description: Additional data added by the origin server but not covered by the ``signatures``. example: {"key": "value"} required: - room_id diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml index 055883c2..08498c35 100644 --- a/api/server-server/events.yaml +++ b/api/server-server/events.yaml @@ -33,37 +33,37 @@ paths: - in: path name: roomId type: string - description: The room ID to get state for + description: The room ID to get state for. required: true x-example: "!abc123:matrix.org" responses: 200: - description: The room state for the room (kept under ``pdus``) + description: The room state for the room (kept under ``pdus``). schema: $ref: "definitions/transaction.yaml" "/event/{eventId}": get: summary: Get a single event description: |- - Retrieves a single event + Retrieves a single event. operationId: getEvent parameters: - in: path name: eventId type: string - description: The event ID to get + description: The event ID to get. required: true x-example: "$abc123:matrix.org" responses: 200: - description: A transaction containing a single PDU which is the event requested + description: A transaction containing a single PDU which is the event requested. schema: $ref: "definitions/transaction.yaml" "/backfill/{roomId}": get: summary: Retrieves the events which precede the given event description: |- - Retreives a sliding-window history of previous PDUs that occurred on the given room. + Retreives a sliding-window history of previous PDUs that occurred in the given room. Starting from the PDU ID(s) given in the ``v`` argument, the PDUs that preceded it are retrived, up to the total number given by the ``limit``. operationId: backfillRoom @@ -71,19 +71,19 @@ paths: - in: path name: roomId type: string - description: The room ID to backfill + description: The room ID to backfill. required: true x-example: "!abc123:matrix.org" - in: query name: v type: string # TODO: The description says this is plural - figure out how to specify multiple, and spec it - description: The event ID to backfill from + description: The event ID to backfill from. required: true x-example: "$abc123:matrix.org" - in: query name: limit type: integer - description: The maximum number of events to retrieve + description: The maximum number of events to retrieve. required: true # TODO: Verify x-example: 10 responses: @@ -91,6 +91,7 @@ paths: description: A transaction containing the PDUs that preceded the given event(s). schema: $ref: "definitions/transaction.yaml" + # TODO: It's possible that this endpoint doesn't exist anymore - verify "/pull": get: summary: Stream events later than a given point in history @@ -101,7 +102,7 @@ paths: - in: query name: v type: string # TODO: The description says this is plural - figure out how to specify multiple, and spec it - description: The event ID to backfill from + description: The event ID to backfill from. required: true x-example: "$abc123:matrix.org" - in: query From 2dae0c9c49542854005291a71d542b9c06fedb59 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 19 Jul 2018 08:54:04 -0600 Subject: [PATCH 202/308] use python3 in dump-swagger.py --- scripts/dump-swagger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dump-swagger.py b/scripts/dump-swagger.py index 7994324f..e02c554e 100755 --- a/scripts/dump-swagger.py +++ b/scripts/dump-swagger.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # dump-swagger reads all of the swagger API docs used in spec generation and # outputs a JSON file which merges them all, for use as input to a swagger UI From 54d6d58fff385981bf8fb60abee083cfd74334a7 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Fri, 20 Jul 2018 14:15:47 +0100 Subject: [PATCH 203/308] naively move content from intro.rst to index.rst --- specification/index.rst | 384 ++++++++++++++++++++++++++++++++++- specification/intro.rst | 400 ------------------------------------- specification/targets.yaml | 3 - 3 files changed, 382 insertions(+), 405 deletions(-) delete mode 100644 specification/intro.rst diff --git a/specification/index.rst b/specification/index.rst index f9a3fef3..cb8b9241 100644 --- a/specification/index.rst +++ b/specification/index.rst @@ -30,18 +30,392 @@ communication ecosystem. To propose a change to the Matrix Spec, see the explanations at `Proposals for Spec Changes to Matrix `_. +.. contents:: Table of Contents +.. sectnum:: + Matrix APIs ----------- The specification consists of the following parts: -`Introduction to Matrix `_ provides a full introduction to Matrix and the spec. - {{apis}} The `Appendices `_ contain supplemental information not specific to one of the above APIs. +Introduction to the Matrix APIs +------------------------------- +.. WARNING:: + The Matrix specification is still evolving: the APIs are not yet frozen + and this document is in places a work in progress or stale. We have made every + effort to clearly flag areas which are still being finalised. + We're publishing it at this point because it's complete enough to be more than + useful and provide a canonical reference to how Matrix is evolving. Our end + goal is to mirror WHATWG's `Living Standard + `_. + +Matrix is a set of open APIs for open-federated Instant Messaging (IM), Voice +over IP (VoIP) and Internet of Things (IoT) communication, designed to create +and support a new global real-time communication ecosystem. The intention is to +provide an open decentralised pubsub layer for the internet for securely +persisting and publishing/subscribing JSON objects. This specification is the +ongoing result of standardising the APIs used by the various components of the +Matrix ecosystem to communicate with one another. + +The principles that Matrix attempts to follow are: + +- Pragmatic Web-friendly APIs (i.e. JSON over REST) +- Keep It Simple & Stupid + + + provide a simple architecture with minimal third-party dependencies. + +- Fully open: + + + Fully open federation - anyone should be able to participate in the global + Matrix network + + Fully open standard - publicly documented standard with no IP or patent + licensing encumbrances + + Fully open source reference implementation - liberally-licensed example + implementations with no IP or patent licensing encumbrances + +- Empowering the end-user + + + The user should be able to choose the server and clients they use + + The user should be control how private their communication is + + The user should know precisely where their data is stored + +- Fully decentralised - no single points of control over conversations or the + network as a whole +- Learning from history to avoid repeating it + + + Trying to take the best aspects of XMPP, SIP, IRC, SMTP, IMAP and NNTP + whilst trying to avoid their failings + + +The functionality that Matrix provides includes: + +- Creation and management of fully distributed chat rooms with no + single points of control or failure +- Eventually-consistent cryptographically secure synchronisation of room + state across a global open network of federated servers and services +- Sending and receiving extensible messages in a room with (optional) + end-to-end encryption +- Extensible user management (inviting, joining, leaving, kicking, banning) + mediated by a power-level based user privilege system. +- Extensible room state management (room naming, aliasing, topics, bans) +- Extensible user profile management (avatars, display names, etc) +- Managing user accounts (registration, login, logout) +- Use of 3rd Party IDs (3PIDs) such as email addresses, phone numbers, + Facebook accounts to authenticate, identify and discover users on Matrix. +- Trusted federation of Identity servers for: + + + Publishing user public keys for PKI + + Mapping of 3PIDs to Matrix IDs + + +The end goal of Matrix is to be a ubiquitous messaging layer for synchronising +arbitrary data between sets of people, devices and services - be that for +instant messages, VoIP call setups, or any other objects that need to be +reliably and persistently pushed from A to B in an interoperable and federated +manner. + + +Spec Change Proposals +~~~~~~~~~~~~~~~~~~~~~ +To propose a change to the Matrix Spec, see the explanations at `Proposals +for Spec Changes to Matrix `_. + + +Architecture +------------ + +Matrix defines APIs for synchronising extensible JSON objects known as +"events" between compatible clients, servers and services. Clients are +typically messaging/VoIP applications or IoT devices/hubs and communicate by +synchronising communication history with their "homeserver" using the +"Client-Server API". Each homeserver stores the communication history and +account information for all of its clients, and shares data with the wider +Matrix ecosystem by synchronising communication history with other homeservers +and their clients. + +Clients typically communicate with each other by emitting events in the +context of a virtual "room". Room data is replicated across *all of the +homeservers* whose users are participating in a given room. As such, *no +single homeserver has control or ownership over a given room*. Homeservers +model communication history as a partially ordered graph of events known as +the room's "event graph", which is synchronised with eventual consistency +between the participating servers using the "Server-Server API". This process +of synchronising shared conversation history between homeservers run by +different parties is called "Federation". Matrix optimises for the +Availability and Partitioned properties of CAP theorem at +the expense of Consistency. + +For example, for client A to send a message to client B, client A performs an +HTTP PUT of the required JSON event on its homeserver (HS) using the +client-server API. A's HS appends this event to its copy of the room's event +graph, signing the message in the context of the graph for integrity. A's HS +then replicates the message to B's HS by performing an HTTP PUT using the +server-server API. B's HS authenticates the request, validates the event's +signature, authorises the event's contents and then adds it to its copy of the +room's event graph. Client B then receives the message from his homeserver via +a long-lived GET request. + +:: + + How data flows between clients + ============================== + + { Matrix client A } { Matrix client B } + ^ | ^ | + | events | Client-Server API | events | + | V | V + +------------------+ +------------------+ + | |---------( HTTPS )--------->| | + | homeserver | | homeserver | + | |<--------( HTTPS )----------| | + +------------------+ Server-Server API +------------------+ + History Synchronisation + (Federation) + + +Users +~~~~~ + +Each client is associated with a user account, which is identified in Matrix +using a unique "user ID". This ID is namespaced to the homeserver which +allocated the account and has the form:: + + @localpart:domain + +See the `appendices `_ for full details of +the structure of user IDs. + +Devices +~~~~~~~ + +The Matrix specification has a particular meaning for the term "device". As a +user, I might have several devices: a desktop client, some web browsers, an +Android device, an iPhone, etc. They broadly relate to a real device in the +physical world, but you might have several browsers on a physical device, or +several Matrix client applications on a mobile device, each of which would be +its own device. + +Devices are used primarily to manage the keys used for end-to-end encryption +(each device gets its own copy of the decryption keys), but they also help +users manage their access - for instance, by revoking access to particular +devices. + +When a user first uses a client, it registers itself as a new device. The +longevity of devices might depend on the type of client. A web client will +probably drop all of its state on logout, and create a new device every time +you log in, to ensure that cryptography keys are not leaked to a new user. In +a mobile client, it might be acceptable to reuse the device if a login session +expires, provided the user is the same. + +Devices are identified by a ``device_id``, which is unique within the scope of +a given user. + +A user may assign a human-readable display name to a device, to help them +manage their devices. + +Events +~~~~~~ + +All data exchanged over Matrix is expressed as an "event". Typically each client +action (e.g. sending a message) correlates with exactly one event. Each event +has a ``type`` which is used to differentiate different kinds of data. ``type`` +values MUST be uniquely globally namespaced following Java's `package naming +conventions`_, e.g. +``com.example.myapp.event``. The special top-level namespace ``m.`` is reserved +for events defined in the Matrix specification - for instance ``m.room.message`` +is the event type for instant messages. Events are usually sent in the context +of a "Room". + +.. _package naming conventions: https://en.wikipedia.org/wiki/Java_package#Package_naming_conventions + +Event Graphs +~~~~~~~~~~~~ + +.. _sect:event-graph: + +Events exchanged in the context of a room are stored in a directed acyclic graph +(DAG) called an "event graph". The partial ordering of this graph gives the +chronological ordering of events within the room. Each event in the graph has a +list of zero or more "parent" events, which refer to any preceding events +which have no chronological successor from the perspective of the homeserver +which created the event. + +Typically an event has a single parent: the most recent message in the room at +the point it was sent. However, homeservers may legitimately race with each +other when sending messages, resulting in a single event having multiple +successors. The next event added to the graph thus will have multiple parents. +Every event graph has a single root event with no parent. + +To order and ease chronological comparison between the events within the graph, +homeservers maintain a ``depth`` metadata field on each event. An event's +``depth`` is a positive integer that is strictly greater than the depths of any +of its parents. The root event should have a depth of 1. Thus if one event is +before another, then it must have a strictly smaller depth. + +Room structure +~~~~~~~~~~~~~~ + +A room is a conceptual place where users can send and receive events. Events are +sent to a room, and all participants in that room with sufficient access will +receive the event. Rooms are uniquely identified internally via "Room IDs", +which have the form:: + + !opaque_id:domain + +There is exactly one room ID for each room. Whilst the room ID does contain a +domain, it is simply for globally namespacing room IDs. The room does NOT +reside on the domain specified. + +See the `appendices `_ for full details of +the structure of a room ID. + +The following conceptual diagram shows an +``m.room.message`` event being sent to the room ``!qporfwt:matrix.org``:: + + { @alice:matrix.org } { @bob:domain.com } + | ^ + | | + [HTTP POST] [HTTP GET] + Room ID: !qporfwt:matrix.org Room ID: !qporfwt:matrix.org + Event type: m.room.message Event type: m.room.message + Content: { JSON object } Content: { JSON object } + | | + V | + +------------------+ +------------------+ + | homeserver | | homeserver | + | matrix.org | | domain.com | + +------------------+ +------------------+ + | ^ + | [HTTP PUT] | + | Room ID: !qporfwt:matrix.org | + | Event type: m.room.message | + | Content: { JSON object } | + `-------> Pointer to the preceding message ------` + PKI signature from matrix.org + Transaction-layer metadata + PKI Authorization header + + ................................... + | Shared Data | + | State: | + | Room ID: !qporfwt:matrix.org | + | Servers: matrix.org, domain.com | + | Members: | + | - @alice:matrix.org | + | - @bob:domain.com | + | Messages: | + | - @alice:matrix.org | + | Content: { JSON object } | + |...................................| + +Federation maintains *shared data structures* per-room between multiple home +servers. The data is split into ``message events`` and ``state events``. + +Message events: + These describe transient 'once-off' activity in a room such as an + instant messages, VoIP call setups, file transfers, etc. They generally + describe communication activity. + +State events: + These describe updates to a given piece of persistent information + ('state') related to a room, such as the room's name, topic, membership, + participating servers, etc. State is modelled as a lookup table of key/value + pairs per room, with each key being a tuple of ``state_key`` and ``event type``. + Each state event updates the value of a given key. + +The state of the room at a given point is calculated by considering all events +preceding and including a given event in the graph. Where events describe the +same state, a merge conflict algorithm is applied. The state resolution +algorithm is transitive and does not depend on server state, as it must +consistently select the same event irrespective of the server or the order the +events were received in. Events are signed by the originating server (the +signature includes the parent relations, type, depth and payload hash) and are +pushed over federation to the participating servers in a room, currently using +full mesh topology. Servers may also request backfill of events over federation +from the other servers participating in a room. + + +Room Aliases +++++++++++++ + +Each room can also have multiple "Room Aliases", which look like:: + + #room_alias:domain + +See the `appendices `_ for full details of +the structure of a room alias. + +A room alias "points" to a room ID and is the human-readable label by which +rooms are publicised and discovered. The room ID the alias is pointing to can +be obtained by visiting the domain specified. Note that the mapping from a room +alias to a room ID is not fixed, and may change over time to point to a +different room ID. For this reason, Clients SHOULD resolve the room alias to a +room ID once and then use that ID on subsequent requests. + +When resolving a room alias the server will also respond with a list of servers +that are in the room that can be used to join via. + +:: + + HTTP GET + #matrix:domain.com !aaabaa:matrix.org + | ^ + | | + _______V____________________|____ + | domain.com | + | Mappings: | + | #matrix >> !aaabaa:matrix.org | + | #golf >> !wfeiofh:sport.com | + | #bike >> !4rguxf:matrix.org | + |________________________________| + +Identity +~~~~~~~~ + +Users in Matrix are identified via their Matrix user ID. However, +existing 3rd party ID namespaces can also be used in order to identify Matrix +users. A Matrix "Identity" describes both the user ID and any other existing IDs +from third party namespaces *linked* to their account. +Matrix users can *link* third-party IDs (3PIDs) such as email addresses, social +network accounts and phone numbers to their user ID. Linking 3PIDs creates a +mapping from a 3PID to a user ID. This mapping can then be used by Matrix +users in order to discover the user IDs of their contacts. +In order to ensure that the mapping from 3PID to user ID is genuine, a globally +federated cluster of trusted "Identity Servers" (IS) are used to verify the 3PID +and persist and replicate the mappings. + +Usage of an IS is not required in order for a client application to be part of +the Matrix ecosystem. However, without one clients will not be able to look up +user IDs using 3PIDs. + + +Profiles +~~~~~~~~ + +Users may publish arbitrary key/value data associated with their account - such +as a human readable display name, a profile photo URL, contact information +(email address, phone numbers, website URLs etc). + +.. TODO + Actually specify the different types of data - e.g. what format are display + names allowed to be? + +Private User Data +~~~~~~~~~~~~~~~~~ + +Users may also store arbitrary private key/value data in their account - such as +client preferences, or server configuration settings which lack any other +dedicated API. The API is symmetrical to managing Profile data. + +.. TODO + Would it really be overengineered to use the same API for both profile & + private user data, but with different ACLs? + Specification Versions ---------------------- @@ -57,3 +431,9 @@ The specification for each API is versioned in the form ``rX.Y.Z``. * A change to ``Z`` represents a change which is backwards-compatible on both sides. Typically this implies a clarification to the specification, rather than a change which must be implemented. + +License +------- + +The Matrix specification is licensed under the `Apache License, Version 2.0 +`_. diff --git a/specification/intro.rst b/specification/intro.rst deleted file mode 100644 index ad545248..00000000 --- a/specification/intro.rst +++ /dev/null @@ -1,400 +0,0 @@ -.. Copyright 2016 OpenMarket Ltd -.. -.. Licensed under the Apache License, Version 2.0 (the "License"); -.. you may not use this file except in compliance with the License. -.. You may obtain a copy of the License at -.. -.. http://www.apache.org/licenses/LICENSE-2.0 -.. -.. Unless required by applicable law or agreed to in writing, software -.. distributed under the License is distributed on an "AS IS" BASIS, -.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -.. See the License for the specific language governing permissions and -.. limitations under the License. - -.. contents:: Table of Contents -.. sectnum:: - -.. Note that this file is specifically unversioned because we don't want to -.. have to add Yet Another version number, and the commentary on what specs we -.. have should hopefully not get complex enough that we need to worry about -.. versioning it. - -Introduction ------------- -.. WARNING:: - The Matrix specification is still evolving: the APIs are not yet frozen - and this document is in places a work in progress or stale. We have made every - effort to clearly flag areas which are still being finalised. - We're publishing it at this point because it's complete enough to be more than - useful and provide a canonical reference to how Matrix is evolving. Our end - goal is to mirror WHATWG's `Living Standard - `_. - -Matrix is a set of open APIs for open-federated Instant Messaging (IM), Voice -over IP (VoIP) and Internet of Things (IoT) communication, designed to create -and support a new global real-time communication ecosystem. The intention is to -provide an open decentralised pubsub layer for the internet for securely -persisting and publishing/subscribing JSON objects. This specification is the -ongoing result of standardising the APIs used by the various components of the -Matrix ecosystem to communicate with one another. - -The principles that Matrix attempts to follow are: - -- Pragmatic Web-friendly APIs (i.e. JSON over REST) -- Keep It Simple & Stupid - - + provide a simple architecture with minimal third-party dependencies. - -- Fully open: - - + Fully open federation - anyone should be able to participate in the global - Matrix network - + Fully open standard - publicly documented standard with no IP or patent - licensing encumbrances - + Fully open source reference implementation - liberally-licensed example - implementations with no IP or patent licensing encumbrances - -- Empowering the end-user - - + The user should be able to choose the server and clients they use - + The user should be control how private their communication is - + The user should know precisely where their data is stored - -- Fully decentralised - no single points of control over conversations or the - network as a whole -- Learning from history to avoid repeating it - - + Trying to take the best aspects of XMPP, SIP, IRC, SMTP, IMAP and NNTP - whilst trying to avoid their failings - - -The functionality that Matrix provides includes: - -- Creation and management of fully distributed chat rooms with no - single points of control or failure -- Eventually-consistent cryptographically secure synchronisation of room - state across a global open network of federated servers and services -- Sending and receiving extensible messages in a room with (optional) - end-to-end encryption -- Extensible user management (inviting, joining, leaving, kicking, banning) - mediated by a power-level based user privilege system. -- Extensible room state management (room naming, aliasing, topics, bans) -- Extensible user profile management (avatars, display names, etc) -- Managing user accounts (registration, login, logout) -- Use of 3rd Party IDs (3PIDs) such as email addresses, phone numbers, - Facebook accounts to authenticate, identify and discover users on Matrix. -- Trusted federation of Identity servers for: - - + Publishing user public keys for PKI - + Mapping of 3PIDs to Matrix IDs - - -The end goal of Matrix is to be a ubiquitous messaging layer for synchronising -arbitrary data between sets of people, devices and services - be that for -instant messages, VoIP call setups, or any other objects that need to be -reliably and persistently pushed from A to B in an interoperable and federated -manner. - - -Spec Change Proposals -~~~~~~~~~~~~~~~~~~~~~ -To propose a change to the Matrix Spec, see the explanations at `Proposals -for Spec Changes to Matrix `_. - - -Architecture ------------- - -Matrix defines APIs for synchronising extensible JSON objects known as -"events" between compatible clients, servers and services. Clients are -typically messaging/VoIP applications or IoT devices/hubs and communicate by -synchronising communication history with their "homeserver" using the -"Client-Server API". Each homeserver stores the communication history and -account information for all of its clients, and shares data with the wider -Matrix ecosystem by synchronising communication history with other homeservers -and their clients. - -Clients typically communicate with each other by emitting events in the -context of a virtual "room". Room data is replicated across *all of the -homeservers* whose users are participating in a given room. As such, *no -single homeserver has control or ownership over a given room*. Homeservers -model communication history as a partially ordered graph of events known as -the room's "event graph", which is synchronised with eventual consistency -between the participating servers using the "Server-Server API". This process -of synchronising shared conversation history between homeservers run by -different parties is called "Federation". Matrix optimises for the -Availability and Partitioned properties of CAP theorem at -the expense of Consistency. - -For example, for client A to send a message to client B, client A performs an -HTTP PUT of the required JSON event on its homeserver (HS) using the -client-server API. A's HS appends this event to its copy of the room's event -graph, signing the message in the context of the graph for integrity. A's HS -then replicates the message to B's HS by performing an HTTP PUT using the -server-server API. B's HS authenticates the request, validates the event's -signature, authorises the event's contents and then adds it to its copy of the -room's event graph. Client B then receives the message from his homeserver via -a long-lived GET request. - -:: - - How data flows between clients - ============================== - - { Matrix client A } { Matrix client B } - ^ | ^ | - | events | Client-Server API | events | - | V | V - +------------------+ +------------------+ - | |---------( HTTPS )--------->| | - | homeserver | | homeserver | - | |<--------( HTTPS )----------| | - +------------------+ Server-Server API +------------------+ - History Synchronisation - (Federation) - - -Users -~~~~~ - -Each client is associated with a user account, which is identified in Matrix -using a unique "user ID". This ID is namespaced to the homeserver which -allocated the account and has the form:: - - @localpart:domain - -See the `appendices `_ for full details of -the structure of user IDs. - -Devices -~~~~~~~ - -The Matrix specification has a particular meaning for the term "device". As a -user, I might have several devices: a desktop client, some web browsers, an -Android device, an iPhone, etc. They broadly relate to a real device in the -physical world, but you might have several browsers on a physical device, or -several Matrix client applications on a mobile device, each of which would be -its own device. - -Devices are used primarily to manage the keys used for end-to-end encryption -(each device gets its own copy of the decryption keys), but they also help -users manage their access - for instance, by revoking access to particular -devices. - -When a user first uses a client, it registers itself as a new device. The -longevity of devices might depend on the type of client. A web client will -probably drop all of its state on logout, and create a new device every time -you log in, to ensure that cryptography keys are not leaked to a new user. In -a mobile client, it might be acceptable to reuse the device if a login session -expires, provided the user is the same. - -Devices are identified by a ``device_id``, which is unique within the scope of -a given user. - -A user may assign a human-readable display name to a device, to help them -manage their devices. - -Events -~~~~~~ - -All data exchanged over Matrix is expressed as an "event". Typically each client -action (e.g. sending a message) correlates with exactly one event. Each event -has a ``type`` which is used to differentiate different kinds of data. ``type`` -values MUST be uniquely globally namespaced following Java's `package naming -conventions`_, e.g. -``com.example.myapp.event``. The special top-level namespace ``m.`` is reserved -for events defined in the Matrix specification - for instance ``m.room.message`` -is the event type for instant messages. Events are usually sent in the context -of a "Room". - -.. _package naming conventions: https://en.wikipedia.org/wiki/Java_package#Package_naming_conventions - -Event Graphs -~~~~~~~~~~~~ - -.. _sect:event-graph: - -Events exchanged in the context of a room are stored in a directed acyclic graph -(DAG) called an "event graph". The partial ordering of this graph gives the -chronological ordering of events within the room. Each event in the graph has a -list of zero or more "parent" events, which refer to any preceding events -which have no chronological successor from the perspective of the homeserver -which created the event. - -Typically an event has a single parent: the most recent message in the room at -the point it was sent. However, homeservers may legitimately race with each -other when sending messages, resulting in a single event having multiple -successors. The next event added to the graph thus will have multiple parents. -Every event graph has a single root event with no parent. - -To order and ease chronological comparison between the events within the graph, -homeservers maintain a ``depth`` metadata field on each event. An event's -``depth`` is a positive integer that is strictly greater than the depths of any -of its parents. The root event should have a depth of 1. Thus if one event is -before another, then it must have a strictly smaller depth. - -Room structure -~~~~~~~~~~~~~~ - -A room is a conceptual place where users can send and receive events. Events are -sent to a room, and all participants in that room with sufficient access will -receive the event. Rooms are uniquely identified internally via "Room IDs", -which have the form:: - - !opaque_id:domain - -There is exactly one room ID for each room. Whilst the room ID does contain a -domain, it is simply for globally namespacing room IDs. The room does NOT -reside on the domain specified. - -See the `appendices `_ for full details of -the structure of a room ID. - -The following conceptual diagram shows an -``m.room.message`` event being sent to the room ``!qporfwt:matrix.org``:: - - { @alice:matrix.org } { @bob:domain.com } - | ^ - | | - [HTTP POST] [HTTP GET] - Room ID: !qporfwt:matrix.org Room ID: !qporfwt:matrix.org - Event type: m.room.message Event type: m.room.message - Content: { JSON object } Content: { JSON object } - | | - V | - +------------------+ +------------------+ - | homeserver | | homeserver | - | matrix.org | | domain.com | - +------------------+ +------------------+ - | ^ - | [HTTP PUT] | - | Room ID: !qporfwt:matrix.org | - | Event type: m.room.message | - | Content: { JSON object } | - `-------> Pointer to the preceding message ------` - PKI signature from matrix.org - Transaction-layer metadata - PKI Authorization header - - ................................... - | Shared Data | - | State: | - | Room ID: !qporfwt:matrix.org | - | Servers: matrix.org, domain.com | - | Members: | - | - @alice:matrix.org | - | - @bob:domain.com | - | Messages: | - | - @alice:matrix.org | - | Content: { JSON object } | - |...................................| - -Federation maintains *shared data structures* per-room between multiple home -servers. The data is split into ``message events`` and ``state events``. - -Message events: - These describe transient 'once-off' activity in a room such as an - instant messages, VoIP call setups, file transfers, etc. They generally - describe communication activity. - -State events: - These describe updates to a given piece of persistent information - ('state') related to a room, such as the room's name, topic, membership, - participating servers, etc. State is modelled as a lookup table of key/value - pairs per room, with each key being a tuple of ``state_key`` and ``event type``. - Each state event updates the value of a given key. - -The state of the room at a given point is calculated by considering all events -preceding and including a given event in the graph. Where events describe the -same state, a merge conflict algorithm is applied. The state resolution -algorithm is transitive and does not depend on server state, as it must -consistently select the same event irrespective of the server or the order the -events were received in. Events are signed by the originating server (the -signature includes the parent relations, type, depth and payload hash) and are -pushed over federation to the participating servers in a room, currently using -full mesh topology. Servers may also request backfill of events over federation -from the other servers participating in a room. - - -Room Aliases -++++++++++++ - -Each room can also have multiple "Room Aliases", which look like:: - - #room_alias:domain - -See the `appendices `_ for full details of -the structure of a room alias. - -A room alias "points" to a room ID and is the human-readable label by which -rooms are publicised and discovered. The room ID the alias is pointing to can -be obtained by visiting the domain specified. Note that the mapping from a room -alias to a room ID is not fixed, and may change over time to point to a -different room ID. For this reason, Clients SHOULD resolve the room alias to a -room ID once and then use that ID on subsequent requests. - -When resolving a room alias the server will also respond with a list of servers -that are in the room that can be used to join via. - -:: - - HTTP GET - #matrix:domain.com !aaabaa:matrix.org - | ^ - | | - _______V____________________|____ - | domain.com | - | Mappings: | - | #matrix >> !aaabaa:matrix.org | - | #golf >> !wfeiofh:sport.com | - | #bike >> !4rguxf:matrix.org | - |________________________________| - -Identity -~~~~~~~~ - -Users in Matrix are identified via their Matrix user ID. However, -existing 3rd party ID namespaces can also be used in order to identify Matrix -users. A Matrix "Identity" describes both the user ID and any other existing IDs -from third party namespaces *linked* to their account. -Matrix users can *link* third-party IDs (3PIDs) such as email addresses, social -network accounts and phone numbers to their user ID. Linking 3PIDs creates a -mapping from a 3PID to a user ID. This mapping can then be used by Matrix -users in order to discover the user IDs of their contacts. -In order to ensure that the mapping from 3PID to user ID is genuine, a globally -federated cluster of trusted "Identity Servers" (IS) are used to verify the 3PID -and persist and replicate the mappings. - -Usage of an IS is not required in order for a client application to be part of -the Matrix ecosystem. However, without one clients will not be able to look up -user IDs using 3PIDs. - - -Profiles -~~~~~~~~ - -Users may publish arbitrary key/value data associated with their account - such -as a human readable display name, a profile photo URL, contact information -(email address, phone numbers, website URLs etc). - -.. TODO - Actually specify the different types of data - e.g. what format are display - names allowed to be? - -Private User Data -~~~~~~~~~~~~~~~~~ - -Users may also store arbitrary private key/value data in their account - such as -client preferences, or server configuration settings which lack any other -dedicated API. The API is symmetrical to managing Profile data. - -.. TODO - Would it really be overengineered to use the same API for both profile & - private user data, but with different ACLs? - -License -------- - -The Matrix specification is licensed under the `Apache License, Version 2.0 -`_. diff --git a/specification/targets.yaml b/specification/targets.yaml index b9718bc4..53957e0a 100644 --- a/specification/targets.yaml +++ b/specification/targets.yaml @@ -2,9 +2,6 @@ targets: index: files: - index.rst - intro: - files: - - intro.rst client_server: files: - client_server_api.rst From e7c79f2bd4a8e6d1bca5c457833e7f7350dd9cbf Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Fri, 20 Jul 2018 15:30:58 +0100 Subject: [PATCH 204/308] avoid .rst link conflict warnings --- specification/index.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/index.rst b/specification/index.rst index cb8b9241..36c89958 100644 --- a/specification/index.rst +++ b/specification/index.rst @@ -187,7 +187,7 @@ allocated the account and has the form:: @localpart:domain -See the `appendices `_ for full details of +See `'Identifier Grammar' the appendices `_ for full details of the structure of user IDs. Devices @@ -271,7 +271,7 @@ There is exactly one room ID for each room. Whilst the room ID does contain a domain, it is simply for globally namespacing room IDs. The room does NOT reside on the domain specified. -See the `appendices `_ for full details of +See `'Identifier Grammar' in the appendices `_ for full details of the structure of a room ID. The following conceptual diagram shows an @@ -347,7 +347,7 @@ Each room can also have multiple "Room Aliases", which look like:: #room_alias:domain -See the `appendices `_ for full details of +See `'Identifier Grammar' in the appendices `_ for full details of the structure of a room alias. A room alias "points" to a room ID and is the human-readable label by which From 59bdcb5615e5716d5b184f725d72ddaf8c02ea3c Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Thu, 19 Jul 2018 21:46:57 +0100 Subject: [PATCH 205/308] Clone and configure the swagger UI as a circle CI artifact Signed-off-by: Stuart Mumford --- .circleci/config.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c11eb70d..3a0b6032 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,12 +4,26 @@ gendoc: &gendoc source /env/bin/activate scripts/gendoc.py -gendoc: &genswagger +genswagger: &genswagger name: Generate the swagger command: | source /env/bin/activate scripts/dump-swagger.py +buildswaggerui: &buildswaggerui + name: Build Swagger UI + command: | + ls scripts/ + mkdir -p api/client-server + git clone https://github.com/matrix-org/swagger-ui swagger-ui + cp -r swagger-ui/dist/* api/client-server/ + mkdir -p api/client-server/json + cp scripts/swagger/api-docs.json api/client-server/json/ + wget https://raw.githubusercontent.com/matrix-org/matrix.org/master/content/swagger.css -O api/client-server/swagger.css + wget https://raw.githubusercontent.com/matrix-org/matrix.org/master/scripts/swagger-ui.patch + patch api/client-server/index.html swagger-ui.patch + + version: 2 jobs: build-docs: @@ -30,11 +44,12 @@ jobs: steps: - checkout - run: *genswagger + - run: *buildswaggerui - store_artifacts: - path: scripts/swagger/api-docs.json + path: api/client-server/ - run: name: "Swagger UI is available at:" - command: SWAGGER_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/scripts/swagger/api-docs.json"; echo "https://matrix.org/docs/api/client-server/?url="$SWAGGER_URL + command: DOCS_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/api/client-server/index.html"; echo $DOCS_URL workflows: version: 2 From 03b74476f12d65ab5fb1d36ae9b25f8bd11d50d5 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 23 Jul 2018 20:20:52 +0100 Subject: [PATCH 206/308] Add config for Giles Signed-off-by: Stuart Mumford --- pyproject.toml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..61f77ec6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,9 @@ +[ tool.giles ] + + [ tool.giles.circleci_artifacts.docs ] + url = "gen/index.html" + message = "Click details to preview the HTML documentation." + + [ tool.giles.circleci_artifacts.random ] + url = "client-server/index.html" + message = "Click to preview the swagger build." From 7c71b697e7b0ac6ccb8227e3557199422bc18a47 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Tue, 24 Jul 2018 11:07:11 +0100 Subject: [PATCH 207/308] Fix typo in Giles config Signed-off-by: Stuart Mumford --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 61f77ec6..b53982b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,6 @@ url = "gen/index.html" message = "Click details to preview the HTML documentation." - [ tool.giles.circleci_artifacts.random ] + [ tool.giles.circleci_artifacts.swagger ] url = "client-server/index.html" message = "Click to preview the swagger build." From 07aeaadef914f68b0e6bca274c8d656da91d52cf Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Tue, 24 Jul 2018 11:41:03 +0100 Subject: [PATCH 208/308] update links which used to point to docs/spec/intro --- specification/client_server_api.rst | 2 +- specification/modules/cas_login.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 8dde4302..1bb989ff 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -1417,7 +1417,7 @@ have to wait in milliseconds before they can try again. .. References .. _`macaroon`: http://research.google.com/pubs/pub41892.html -.. _`devices`: ../intro.html#devices +.. _`devices`: ../index.html#devices .. Links through the external API docs are below .. ============================================= diff --git a/specification/modules/cas_login.rst b/specification/modules/cas_login.rst index b651431a..5de98057 100644 --- a/specification/modules/cas_login.rst +++ b/specification/modules/cas_login.rst @@ -98,9 +98,9 @@ check for certain user attributes in the response. Any required attributes should be configured by the server administrator. Once the ticket has been validated, the server MUST map the CAS ``user_id`` -to a valid `Matrix user identifier <../intro.html#user-identifiers>`_. The +to a valid `Matrix user identifier <../index.html#user-identifiers>`_. The guidance in `Mapping from other character sets -<../intro.html#mapping-from-other-character-sets>`_ may be useful. +<../index.html#mapping-from-other-character-sets>`_ may be useful. If the generated user identifier represents a new user, it should be registered as a new user. From 26505533dd497859ae69b18ecd008fa56a14d563 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Tue, 24 Jul 2018 16:06:59 +0100 Subject: [PATCH 209/308] drop dns_name = host in favour of host --- specification/appendices/identifier_grammar.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/specification/appendices/identifier_grammar.rst b/specification/appendices/identifier_grammar.rst index e85bf410..3abcfae2 100644 --- a/specification/appendices/identifier_grammar.rst +++ b/specification/appendices/identifier_grammar.rst @@ -25,8 +25,7 @@ number of identifiers, as described below. The server name represents the address at which the homeserver in question can be reached by other homeservers. The complete grammar is:: - server_name = dns_name [ ":" port] - dns_name = host + server_name = host [ ":" port] port = *DIGIT where ``host`` is as defined by `RFC3986, section 3.2.2 From 21c4af70a95b3bb7d7f8767e91487a5df5475a17 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 09:13:15 -0600 Subject: [PATCH 210/308] Fix schema definitions (spelling, attributes) --- api/server-server/definitions/edu.yaml | 5 ++++- api/server-server/definitions/transaction.yaml | 2 +- api/server-server/definitions/unsigned_pdu.yaml | 1 - api/server-server/events.yaml | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/api/server-server/definitions/edu.yaml b/api/server-server/definitions/edu.yaml index af12752c..c89573fe 100644 --- a/api/server-server/definitions/edu.yaml +++ b/api/server-server/definitions/edu.yaml @@ -24,14 +24,17 @@ properties: type: string description: The type of ephemeral message. example: "m.presence" + required: true origin: type: string description: The server name sending the ephemeral message. - example: "matrix.org" + example: "matrix.org" + required: true destination: type: string description: The server name receiving the ephemeral message. example: "elsewhere.com" + required: true content: type: object description: The content of the ephemeral message. diff --git a/api/server-server/definitions/transaction.yaml b/api/server-server/definitions/transaction.yaml index e4c1f559..930ddec1 100644 --- a/api/server-server/definitions/transaction.yaml +++ b/api/server-server/definitions/transaction.yaml @@ -20,7 +20,7 @@ properties: origin: type: string description: |- - The ``server_name`` of the hoemserver sending this transaction. + The ``server_name`` of the homeserver sending this transaction. example: "example.org" origin_server_ts: type: integer diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml index ca0a1183..011b2736 100644 --- a/api/server-server/definitions/unsigned_pdu.yaml +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -37,7 +37,6 @@ properties: type: type: string description: Event type - required: true example: "m.room.message" state_key: type: string diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml index 08498c35..96b2a825 100644 --- a/api/server-server/events.yaml +++ b/api/server-server/events.yaml @@ -65,7 +65,7 @@ paths: description: |- Retreives a sliding-window history of previous PDUs that occurred in the given room. Starting from the PDU ID(s) given in the ``v`` argument, the PDUs that preceded it - are retrived, up to the total number given by the ``limit``. + are retrieved, up to the total number given by the ``limit``. operationId: backfillRoom parameters: - in: path @@ -84,7 +84,7 @@ paths: name: limit type: integer description: The maximum number of events to retrieve. - required: true # TODO: Verify + required: true x-example: 10 responses: 200: From 88beaf4ad8f5a27bbdd5a3c30d0fd002ab431954 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 09:13:46 -0600 Subject: [PATCH 211/308] Completely remove /pull It's not been in use for years - we can safely let it disappear --- api/server-server/events.yaml | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml index 96b2a825..d540085d 100644 --- a/api/server-server/events.yaml +++ b/api/server-server/events.yaml @@ -91,29 +91,3 @@ paths: description: A transaction containing the PDUs that preceded the given event(s). schema: $ref: "definitions/transaction.yaml" - # TODO: It's possible that this endpoint doesn't exist anymore - verify - "/pull": - get: - summary: Stream events later than a given point in history - description: |- - Retrieves all of the transactions later than any version given by the ``v`` arguments. - operationId: pull - parameters: - - in: query - name: v - type: string # TODO: The description says this is plural - figure out how to specify multiple, and spec it - description: The event ID to backfill from. - required: true - x-example: "$abc123:matrix.org" - - in: query - name: origin - type: string - description: The origin # TODO: What is this actually? - required: true # TODO: Verify - x-example: "matrix.org" - responses: - 200: - # TODO: This could do with a better description - description: A transaction containing multiple PDUs - schema: - $ref: "definitions/transaction.yaml" \ No newline at end of file From a4e5a461f9b5886b5183f28072916c0a0cbba3c1 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 09:14:31 -0600 Subject: [PATCH 212/308] Improve handing of lists appearing in items --- scripts/templating/matrix_templates/units.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index fc2b9181..c99b7c38 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -140,9 +140,6 @@ def inherit_parents(obj): """ logger.debug("inherit_parents %r" % obj) - if isinstance(obj, list): - return [inherit_parents(obj[i]) for i in range(0, len(obj))] - parents = obj.get("allOf", []) if not parents: return obj @@ -298,6 +295,8 @@ def process_data_type(prop, required=False, enforce_title=True): elif prop_type == "array": items = prop["items"] + # Items can be a list of schemas or a schema itself + # http://json-schema.org/latest/json-schema-validation.html#rfc.section.6.4 if isinstance(items, list): prop_title = "[" for i in items: @@ -385,9 +384,6 @@ def get_example_for_schema(schema): """Returns a python object representing a suitable example for this object""" schema = inherit_parents(schema) - if isinstance(schema, list): - return [get_example_for_schema(v) for v in schema] - if 'example' in schema: example = schema['example'] return example @@ -407,10 +403,13 @@ def get_example_for_schema(schema): if proptype == 'array': if 'items' not in schema: raise Exception('"array" property has neither items nor example') - result = get_example_for_schema(schema['items']) + items = schema['items'] + if isinstance(items, list): + return [get_example_for_schema(i) for i in items] + result = get_example_for_schema(items) if isinstance(result, list): - return result - return [result] + return [result] + return result if proptype == 'integer': return 0 From 042772aaf58141c9b5a433134c88b0817e8af648 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 10:13:23 -0600 Subject: [PATCH 213/308] Make nested titles better --- scripts/templating/matrix_templates/units.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index c99b7c38..e6493f2e 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -298,12 +298,12 @@ def process_data_type(prop, required=False, enforce_title=True): # Items can be a list of schemas or a schema itself # http://json-schema.org/latest/json-schema-validation.html#rfc.section.6.4 if isinstance(items, list): - prop_title = "[" + nested_titles = [] for i in items: nested = process_data_type(i) tables.extend(nested['tables']) - prop_title = "%s%s, " % (prop_title, nested['title']) - prop_title = prop_title[:-2] + "]" + nested_titles.extend([nested['title']]) + prop_title = "[%s]" % (", ".join(nested_titles), ) else: nested = process_data_type(prop["items"]) prop_title = "[%s]" % nested["title"] From 5ade15534d128519a6af77777e76aa79e466fa56 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 10:17:39 -0600 Subject: [PATCH 214/308] Remove extra branch in example generation --- scripts/templating/matrix_templates/units.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index e6493f2e..15c987fd 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -406,10 +406,7 @@ def get_example_for_schema(schema): items = schema['items'] if isinstance(items, list): return [get_example_for_schema(i) for i in items] - result = get_example_for_schema(items) - if isinstance(result, list): - return [result] - return result + return get_example_for_schema(items) if proptype == 'integer': return 0 From 9fa838d3e8d51103ac990ed4808274441149e00f Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 10:17:57 -0600 Subject: [PATCH 215/308] Improve type naming --- scripts/templating/matrix_templates/units.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 15c987fd..a4f682bb 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -302,7 +302,7 @@ def process_data_type(prop, required=False, enforce_title=True): for i in items: nested = process_data_type(i) tables.extend(nested['tables']) - nested_titles.extend([nested['title']]) + nested_titles.append(nested['title']) prop_title = "[%s]" % (", ".join(nested_titles), ) else: nested = process_data_type(prop["items"]) @@ -523,10 +523,8 @@ class MatrixUnits(Units): items = param.get("items") if items: if isinstance(items, list): - val_type = "[" - for i in items: - val_type += "%s, " % items.get("type") - val_type = val_type[:-2] + "]" + types = ", ".join(i.get("type") for i in items) + val_type = "[%s]" % (types,) else: val_type = "[%s]" % items.get("type") From acf9632afcd0fb3861c203a1fb346ffaaffa98d8 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 10:25:24 -0600 Subject: [PATCH 216/308] Enlist the examples --- scripts/templating/matrix_templates/units.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index a4f682bb..5b39a2b3 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -406,7 +406,7 @@ def get_example_for_schema(schema): items = schema['items'] if isinstance(items, list): return [get_example_for_schema(i) for i in items] - return get_example_for_schema(items) + return [get_example_for_schema(items)] if proptype == 'integer': return 0 From f09c4fd286e344512ae1890650c5d7bcb0bb022c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 21:35:12 -0600 Subject: [PATCH 217/308] Convert joins to swagger --- api/server-server/joins.yaml | 275 ++++++++++++++++++++++++++++ specification/server_server_api.rst | 11 +- 2 files changed, 276 insertions(+), 10 deletions(-) create mode 100644 api/server-server/joins.yaml diff --git a/api/server-server/joins.yaml b/api/server-server/joins.yaml new file mode 100644 index 00000000..d4a7ed9e --- /dev/null +++ b/api/server-server/joins.yaml @@ -0,0 +1,275 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Join Room API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/make_join/{roomId}/{userId}": + get: + summary: Get information required to make a join event for a room + description: |- + Asks the receiving server to return information that the sending + server will need to prepare a join event to get into the room. + This is part of the `Joining Rooms`_ handshake. + operationId: makeJoin + parameters: + - in: path + name: roomId + type: string + description: The room ID that is about to be joined + required: true + x-example: "!abc123:matrix.org" + - in: path + name: userId + type: string + description: The user ID the join event will be for + required: true + x-example: "@someone:example.org" + responses: + 200: + description: |- + An unsigned event that the server may now use as a template + for the rest of the `Joining Rooms`_ handshake. + schema: + allOf: + - $ref: "definitions/unsigned_pdu.yaml" + - type: object + properties: + # Note: we override a bunch of parameters to change their descriptions + sender: + type: string + description: The user ID of the joining member + example: "@someone:example.org" + origin: + type: string + description: The name of the resident homeserver + example: "matrix.org" + origin_server_ts: + type: integer + format: int64 + description: A timestamp added by the resident homeserver + example: 1234567890 + type: + type: string + description: The value ``m.room.member`` + required: true + example: "m.room.member" + state_key: + type: string + description: The user ID of the joining member + example: "@someone:example.org" + content: + type: object + title: Membership Event Content + description: The content of the event + example: {"membership": "join"} + properties: + membership: + type: string + description: The value ``join`` + example: "join" + required: ['membership'] + depth: + type: integer + description: This field must be present but is ignored; it may be 0 + example: 12 + auth_events: + type: array + description: |- + An event reference list containing the authorization events that would + allow the member to join the room + items: + type: array + maxItems: 2 + minItems: 2 + items: + - type: string + title: Event ID + example: "$abc123:matrix.org" + - type: object + title: Event Hash + example: { + "sha256": "abase64encodedsha256hashshouldbe43byteslong" + } + properties: + sha256: + type: string + description: The event hash + example: abase64encodedsha256hashshouldbe43byteslong + required: ['sha256'] + redacts: + type: string + description: Not used + required: + # Every other field is already flagged as required by the $ref + - state_key + examples: + application/json: { + "$ref": "examples/unsigned_pdu.json", + "type": "m.room.member", + "content": { + "membership": "join" + } + } + "/send_join/{roomId}/{eventId}": + put: + summary: Submit a signed join event to a resident server + description: |- + Submits a signed join event to the resident server for it + to accept it into the room's graph. + operationId: sendJoin + parameters: + - in: path + name: roomId + type: string + description: The room ID that is about to be joined + required: true + x-example: "!abc123:matrix.org" + - in: path + name: eventId + type: string + description: The event ID for the join event + required: true + x-example: "$abc123:example.org" + - in: body + name: body + type: object + required: true + schema: + allOf: + - $ref: "definitions/pdu.yaml" + - type: object + properties: + # Note: we override a bunch of parameters to change their descriptions + sender: + type: string + description: The user ID of the joining member + example: "@someone:example.org" + origin: + type: string + description: The name of the joining homeserver + example: "matrix.org" + origin_server_ts: + type: integer + format: int64 + description: A timestamp added by the joining homeserver + example: 1234567890 + type: + type: string + description: The value ``m.room.member`` + example: "m.room.member" + state_key: + type: string + description: The user ID of the joining member + example: "@someone:example.org" + content: + type: object + title: Membership Event Content + description: The content of the event + example: {"membership": "join"} + properties: + membership: + type: string + description: The value ``join`` + example: "join" + required: ['membership'] + depth: + type: integer + description: This field must be present but is ignored; it may be 0 + example: 12 + auth_events: + type: array + description: |- + An event reference list containing the authorization events that would + allow the member to join the room + items: + type: array + maxItems: 2 + minItems: 2 + items: + - type: string + title: Event ID + example: "$abc123:matrix.org" + - type: object + title: Event Hash + example: { + "sha256": "abase64encodedsha256hashshouldbe43byteslong" + } + properties: + sha256: + type: string + description: The event hash + example: abase64encodedsha256hashshouldbe43byteslong + required: ['sha256'] + redacts: + type: string + description: Not used + required: + # Every other field is already flagged as required by the $ref + - state_key + example: { + "$ref": "examples/pdu.json", + "type": "m.room.member", + "content": { + "membership": "join" + } + } + responses: + 200: + description: |- + The full state for the room, having accepted the join event + schema: + type: array + minItems: 2 + maxItems: 2 + items: + - type: integer + description: The value ``200`` + example: 200 + - type: object + title: Room State + description: The state for the room + properties: + auth_chain: + type: array + description: The auth chain + items: + type: object + properties: {} + # TODO: Verify schema + state: + type: array + description: The room state + items: + type: object + properties: {} + # TODO: Verify schema + required: ["auth_chain", "state"] + examples: + application/json: [ + 200, + { + # TODO: Use the appropriate refs (see TODOs in schema) + "auth_chain": [], + "state": [] + } + ] diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 6f980045..e8960d11 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -689,16 +689,7 @@ All these URLs are name-spaced within a prefix of:: {{query_general_ss_http_api}} -To join a room:: - - GET .../make_join// - Response: JSON encoding of a join proto-event - - PUT .../send_join// - Response: JSON encoding of the state of the room at the time of the event - -Performs the room join handshake. For more information, see "Joining Rooms" -below. +{{joins_ss_http_api}} Joining Rooms ------------- From 004998b98f6ec2e1b15c49e3c27674d92ddf6a28 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 12 Jul 2018 22:10:46 -0600 Subject: [PATCH 218/308] Convert invites to swagger The whole section reads like a description for the endpoint, and has been replaced by the swagger definition now (rather than at a later stage). All the same information should be kept. --- .../definitions/invite_event.yaml | 88 +++++++++++++++++++ api/server-server/invites.yaml | 84 ++++++++++++++++++ specification/server_server_api.rst | 60 +------------ 3 files changed, 173 insertions(+), 59 deletions(-) create mode 100644 api/server-server/definitions/invite_event.yaml create mode 100644 api/server-server/invites.yaml diff --git a/api/server-server/definitions/invite_event.yaml b/api/server-server/definitions/invite_event.yaml new file mode 100644 index 00000000..989c159f --- /dev/null +++ b/api/server-server/definitions/invite_event.yaml @@ -0,0 +1,88 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +type: object +title: Invite Event +description: An invite event +allOf: + - $ref: "pdu.yaml" + - type: object + properties: + # Note: we override a bunch of parameters to change their descriptions + sender: + type: string + # TODO: Verify/clarify this - it doesn't seem right, given this is a 'regular' invite + description: |- + The matrix ID of the user who sent the original ``m.room.third_party_invite`` + example: "@someone:example.org" + origin: + type: string + description: The name of the inviting homeserver + example: "matrix.org" + origin_server_ts: + type: integer + format: int64 + description: A timestamp added by the inviting homeserver + example: 1234567890 + type: + type: string + description: The value ``m.room.member`` + example: "m.room.member" + state_key: + type: string + description: The user ID of the invited member + example: "@joe:elsewhere.com" + content: + type: object + title: Membership Event Content + description: |- + The content of the event, matching what is available in the + `Client-Server API`_ + example: {"membership": "invite"} + properties: + membership: + type: string + description: The value ``invite`` + example: "invite" + required: ['membership'] + auth_events: + type: array + description: |- + An event reference list containing the authorization events that would + allow the member to be invited to the room + items: + type: array + maxItems: 2 + minItems: 2 + items: + - type: string + title: Event ID + example: "$abc123:matrix.org" + - type: object + title: Event Hash + example: { + "sha256": "abase64encodedsha256hashshouldbe43byteslong" + } + properties: + sha256: + type: string + description: The event hash + example: abase64encodedsha256hashshouldbe43byteslong + required: ['sha256'] + redacts: + type: string + description: Not used + required: + # Every other field is already flagged as required by the $ref + - state_key + - unsigned # TODO: apparently this is required? diff --git a/api/server-server/invites.yaml b/api/server-server/invites.yaml new file mode 100644 index 00000000..ad20241b --- /dev/null +++ b/api/server-server/invites.yaml @@ -0,0 +1,84 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Invite User To Room API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/invite/{roomId}/{eventId}": + put: + summary: Invites a user to a room + description: |- + Invites a remote user to a room. Once the event has been + signed by both the inviting homeserver and the invited + homeserver, it can be sent to all of the users in the room. + operationId: sendInvite + parameters: + - in: path + name: roomId + type: string + description: The room ID that the user is being invited to + required: true + x-example: "!abc123:matrix.org" + - in: path + name: eventId + type: string + description: The event ID for the invite event + required: true + x-example: "$abc123:example.org" + - in: body + name: body + type: object + required: true + schema: + $ref: "definitions/invite_event.yaml" + example: { + "$ref": "examples/pdu.json", + "type": "m.room.member", + "content": { + "membership": "invite" + } + } + responses: + 200: + description: |- + The event with the invited server's signature added. All other fields of the events + should remain untouched. + schema: + type: array + minItems: 2 + maxItems: 2 + items: + - type: integer + description: The value ``200`` + example: 200 + - $ref: "definitions/invite_event.yaml" + examples: + application/json: [ + 200, + { + "$ref": "examples/pdu.json", + "type": "m.room.member", + "content": { + "membership": "invite" + } + } + ] diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index e8960d11..f1825f27 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -879,65 +879,7 @@ that requested by the requester in the ``v`` parameter). Inviting to a room ------------------ -When a user wishes to invite another user to a local room and the other user is -on a different server, the inviting server will send a request to the invited -server:: - - PUT .../invite/{roomId}/{eventId} - -The required fields in the JSON body are: - -======================== ============ ========================================= - Key Type Description -======================== ============ ========================================= -``room_id`` String The room ID of the room. Must be the same - as the room ID specified in the path. -``event_id`` String The ID of the event. Must be the same as - the event ID specified in the path. -``type`` String The value ``m.room.member``. -``auth_events`` List An event-reference list containing the - IDs of the authorization events that - would allow this member to be invited in - the room. -``content`` Object The content of the event. -``depth`` Integer The depth of the event. -``origin`` String The name of the inviting homeserver. -``origin_server_ts`` Integer A timestamp added by the inviting - homeserver. -``prev_events`` List An event-reference list containing the - IDs of the immediate predecessor events. -``sender`` String The Matrix ID of the user who sent the - original ``m.room.third_party_invite``. -``state_key`` String The Matrix ID of the invited user. -``signatures`` Object The signature of the event from the - origin server. -``unsigned`` Object An object containing the properties that - aren't part of the signature's - computation. -======================== ============ ========================================= - -Where the ``content`` key contains the content for the ``m.room.member`` event -specified in the `Client-Server API`_. Note that the ``membership`` property of -the content must be ``invite``. - -Upon receiving this request, the invited homeserver will append its signature to -the event and respond to the request with the following JSON body:: - - [ - 200, - "event": {...} - ] - -Where ``event`` contains the event signed by both homeservers, using the same -JSON keys as the initial request on ``/invite/{roomId}/{eventId}``. Note that, -except for the ``signatures`` object (which now contains an additional signature), -all of the event's keys remain the same as in the event initially provided. - -This response format is due to a typo in Synapse, the first implementation of -Matrix's APIs, and is preserved to maintain compatibility. - -Now that the event has been signed by both the inviting homeserver and the -invited homeserver, it can be sent to all of the users in the room. +{{invites_ss_http_api}} Third-party invites ------------------- From 9c1c541a3829642b3371d8523d0fd3f13c095e8e Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 13 Jul 2018 15:04:22 -0600 Subject: [PATCH 219/308] Remove extra "required" declarations "required" is something that is handled as an array, and these are invalid. The `type` in both cases is covered by a required array elsewhere --- api/server-server/joins.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/api/server-server/joins.yaml b/api/server-server/joins.yaml index d4a7ed9e..09714724 100644 --- a/api/server-server/joins.yaml +++ b/api/server-server/joins.yaml @@ -71,7 +71,6 @@ paths: type: type: string description: The value ``m.room.member`` - required: true example: "m.room.member" state_key: type: string From 45ec8269e68c3c0f033f5414ff3e45e763ff7236 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 13 Jul 2018 15:04:45 -0600 Subject: [PATCH 220/308] Fill out the required fields in all the examples --- api/server-server/examples/unsigned_pdu.json | 7 +++++++ api/server-server/invites.yaml | 8 ++++++-- api/server-server/joins.yaml | 2 ++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/api/server-server/examples/unsigned_pdu.json b/api/server-server/examples/unsigned_pdu.json index 6ed4b79d..0b585a63 100644 --- a/api/server-server/examples/unsigned_pdu.json +++ b/api/server-server/examples/unsigned_pdu.json @@ -4,6 +4,13 @@ "origin": "example.com", "event_id": "$a4ecee13e2accdadf56c1025:example.com", "origin_server_ts": 1404838188000, + "depth": 12, + "auth_events": [ + [ + "$af232176:example.org", + {"sha256": "abase64encodedsha256hashshouldbe43byteslong"} + ] + ], "type": "m.room.message", "prev_events": [ [ diff --git a/api/server-server/invites.yaml b/api/server-server/invites.yaml index ad20241b..d176a5ae 100644 --- a/api/server-server/invites.yaml +++ b/api/server-server/invites.yaml @@ -53,9 +53,11 @@ paths: example: { "$ref": "examples/pdu.json", "type": "m.room.member", + "state_key": "@someone:example.org", "content": { "membership": "invite" - } + }, + "unsigned": {} } responses: 200: @@ -77,8 +79,10 @@ paths: { "$ref": "examples/pdu.json", "type": "m.room.member", + "state_key": "@someone:example.org", "content": { "membership": "invite" - } + }, + "unsigned": {} } ] diff --git a/api/server-server/joins.yaml b/api/server-server/joins.yaml index 09714724..b3aebf14 100644 --- a/api/server-server/joins.yaml +++ b/api/server-server/joins.yaml @@ -125,6 +125,7 @@ paths: application/json: { "$ref": "examples/unsigned_pdu.json", "type": "m.room.member", + "state_key": "@someone:example.org", "content": { "membership": "join" } @@ -228,6 +229,7 @@ paths: example: { "$ref": "examples/pdu.json", "type": "m.room.member", + "state_key": "@someone:example.org", "content": { "membership": "join" } From 7314405904b507290d74188c783e8f3b6db7c293 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 13 Jul 2018 15:04:52 -0600 Subject: [PATCH 221/308] Update comment --- api/server-server/definitions/invite_event.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server-server/definitions/invite_event.yaml b/api/server-server/definitions/invite_event.yaml index 989c159f..d87a1daa 100644 --- a/api/server-server/definitions/invite_event.yaml +++ b/api/server-server/definitions/invite_event.yaml @@ -85,4 +85,4 @@ allOf: required: # Every other field is already flagged as required by the $ref - state_key - - unsigned # TODO: apparently this is required? + - unsigned # TODO: apparently this is required? Verify. From 797aca87bd28c3327a4b855f9dd572cfe02b4373 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 13 Jul 2018 15:05:44 -0600 Subject: [PATCH 222/308] Update the check_examples.py script to resolve references The script previously wasn't chasing down valid references in examples. --- api/check_examples.py | 45 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/api/check_examples.py b/api/check_examples.py index be0676bb..fb9430b6 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -56,7 +56,8 @@ def check_parameter(filepath, request, parameter): # Setting the 'id' tells jsonschema where the file is so that it # can correctly resolve relative $ref references in the schema schema['id'] = fileurl - resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_yaml}) + example = resolve_references(filepath, example) + resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file}) jsonschema.validate(example, schema, resolver=resolver) except Exception as e: raise ValueError("Error validating JSON schema for %r" % ( @@ -76,7 +77,8 @@ def check_response(filepath, request, code, response): # Setting the 'id' tells jsonschema where the file is so that it # can correctly resolve relative $ref references in the schema schema['id'] = fileurl - resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_yaml}) + example = resolve_references(filepath, example) + resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file}) jsonschema.validate(example, schema, resolver=resolver) except Exception as e: raise ValueError("Error validating JSON schema for %r %r" % ( @@ -104,6 +106,27 @@ def check_swagger_file(filepath): check_response(filepath, request, code, response) +def resolve_references(path, schema): + if isinstance(schema, dict): + # do $ref first + if '$ref' in schema: + value = schema['$ref'] + path = os.path.abspath(os.path.join(os.path.dirname(path), value)) + ref = load_file("file://" + path) + result = resolve_references(path, ref) + del schema['$ref'] + else: + result = {} + + for key, value in schema.items(): + result[key] = resolve_references(path, value) + return result + elif isinstance(schema, list): + return [resolve_references(path, value) for value in schema] + else: + return schema + + def load_yaml(path): if not path.startswith("file:///"): raise Exception("Bad ref: %s" % (path,)) @@ -112,6 +135,24 @@ def load_yaml(path): return yaml.load(f) +def load_json(path): + if not path.startswith("file:///"): + raise Exception("Bad ref: %s" % (path,)) + path = path[len("file://"):] + with open(path, "r") as f: + return json.load(f) + + +def load_file(path): + print("Loading reference: %s" % path) + if path.endswith(".json"): + return load_json(path) + else: + # We have to assume it's YAML because some of the YAML examples + # do not have file extensions. + return load_yaml(path) + + if __name__ == '__main__': paths = sys.argv[1:] if not paths: From eaf175056f48ca137dbde4c5d27f771b680b3c78 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 18 Jul 2018 10:59:04 -0600 Subject: [PATCH 223/308] Full stops --- .../definitions/invite_event.yaml | 20 +++---- api/server-server/invites.yaml | 6 +- api/server-server/joins.yaml | 60 +++++++++---------- 3 files changed, 43 insertions(+), 43 deletions(-) diff --git a/api/server-server/definitions/invite_event.yaml b/api/server-server/definitions/invite_event.yaml index d87a1daa..9cef8061 100644 --- a/api/server-server/definitions/invite_event.yaml +++ b/api/server-server/definitions/invite_event.yaml @@ -23,43 +23,43 @@ allOf: type: string # TODO: Verify/clarify this - it doesn't seem right, given this is a 'regular' invite description: |- - The matrix ID of the user who sent the original ``m.room.third_party_invite`` + The matrix ID of the user who sent the original ``m.room.third_party_invite``. example: "@someone:example.org" origin: type: string - description: The name of the inviting homeserver + description: The name of the inviting homeserver. example: "matrix.org" origin_server_ts: type: integer format: int64 - description: A timestamp added by the inviting homeserver + description: A timestamp added by the inviting homeserver. example: 1234567890 type: type: string - description: The value ``m.room.member`` + description: The value ``m.room.member``. example: "m.room.member" state_key: type: string - description: The user ID of the invited member + description: The user ID of the invited member. example: "@joe:elsewhere.com" content: type: object title: Membership Event Content description: |- The content of the event, matching what is available in the - `Client-Server API`_ + `Client-Server API`_. example: {"membership": "invite"} properties: membership: type: string - description: The value ``invite`` + description: The value ``invite``. example: "invite" required: ['membership'] auth_events: type: array description: |- An event reference list containing the authorization events that would - allow the member to be invited to the room + allow the member to be invited to the room. items: type: array maxItems: 2 @@ -76,12 +76,12 @@ allOf: properties: sha256: type: string - description: The event hash + description: The event hash. example: abase64encodedsha256hashshouldbe43byteslong required: ['sha256'] redacts: type: string - description: Not used + description: Not used. required: # Every other field is already flagged as required by the $ref - state_key diff --git a/api/server-server/invites.yaml b/api/server-server/invites.yaml index d176a5ae..a5d21938 100644 --- a/api/server-server/invites.yaml +++ b/api/server-server/invites.yaml @@ -35,13 +35,13 @@ paths: - in: path name: roomId type: string - description: The room ID that the user is being invited to + description: The room ID that the user is being invited to. required: true x-example: "!abc123:matrix.org" - in: path name: eventId type: string - description: The event ID for the invite event + description: The event ID for the invite event. required: true x-example: "$abc123:example.org" - in: body @@ -70,7 +70,7 @@ paths: maxItems: 2 items: - type: integer - description: The value ``200`` + description: The value ``200``. example: 200 - $ref: "definitions/invite_event.yaml" examples: diff --git a/api/server-server/joins.yaml b/api/server-server/joins.yaml index b3aebf14..11a47af6 100644 --- a/api/server-server/joins.yaml +++ b/api/server-server/joins.yaml @@ -35,13 +35,13 @@ paths: - in: path name: roomId type: string - description: The room ID that is about to be joined + description: The room ID that is about to be joined. required: true x-example: "!abc123:matrix.org" - in: path name: userId type: string - description: The user ID the join event will be for + description: The user ID the join event will be for. required: true x-example: "@someone:example.org" responses: @@ -57,29 +57,29 @@ paths: # Note: we override a bunch of parameters to change their descriptions sender: type: string - description: The user ID of the joining member + description: The user ID of the joining member. example: "@someone:example.org" origin: type: string - description: The name of the resident homeserver + description: The name of the resident homeserver. example: "matrix.org" origin_server_ts: type: integer format: int64 - description: A timestamp added by the resident homeserver + description: A timestamp added by the resident homeserver. example: 1234567890 type: type: string - description: The value ``m.room.member`` + description: The value ``m.room.member``. example: "m.room.member" state_key: type: string - description: The user ID of the joining member + description: The user ID of the joining member. example: "@someone:example.org" content: type: object title: Membership Event Content - description: The content of the event + description: The content of the event. example: {"membership": "join"} properties: membership: @@ -89,13 +89,13 @@ paths: required: ['membership'] depth: type: integer - description: This field must be present but is ignored; it may be 0 + description: This field must be present but is ignored; it may be 0. example: 12 auth_events: type: array description: |- An event reference list containing the authorization events that would - allow the member to join the room + allow the member to join the room. items: type: array maxItems: 2 @@ -112,12 +112,12 @@ paths: properties: sha256: type: string - description: The event hash + description: The event hash. example: abase64encodedsha256hashshouldbe43byteslong required: ['sha256'] redacts: type: string - description: Not used + description: Not used. required: # Every other field is already flagged as required by the $ref - state_key @@ -141,13 +141,13 @@ paths: - in: path name: roomId type: string - description: The room ID that is about to be joined + description: The room ID that is about to be joined. required: true x-example: "!abc123:matrix.org" - in: path name: eventId type: string - description: The event ID for the join event + description: The event ID for the join event. required: true x-example: "$abc123:example.org" - in: body @@ -162,45 +162,45 @@ paths: # Note: we override a bunch of parameters to change their descriptions sender: type: string - description: The user ID of the joining member + description: The user ID of the joining member. example: "@someone:example.org" origin: type: string - description: The name of the joining homeserver + description: The name of the joining homeserver. example: "matrix.org" origin_server_ts: type: integer format: int64 - description: A timestamp added by the joining homeserver + description: A timestamp added by the joining homeserver. example: 1234567890 type: type: string - description: The value ``m.room.member`` + description: The value ``m.room.member``. example: "m.room.member" state_key: type: string - description: The user ID of the joining member + description: The user ID of the joining member. example: "@someone:example.org" content: type: object title: Membership Event Content - description: The content of the event + description: The content of the event. example: {"membership": "join"} properties: membership: type: string - description: The value ``join`` + description: The value ``join``. example: "join" required: ['membership'] depth: type: integer - description: This field must be present but is ignored; it may be 0 + description: This field must be present but is ignored; it may be 0. example: 12 auth_events: type: array description: |- An event reference list containing the authorization events that would - allow the member to join the room + allow the member to join the room. items: type: array maxItems: 2 @@ -217,12 +217,12 @@ paths: properties: sha256: type: string - description: The event hash + description: The event hash. example: abase64encodedsha256hashshouldbe43byteslong required: ['sha256'] redacts: type: string - description: Not used + description: Not used. required: # Every other field is already flagged as required by the $ref - state_key @@ -237,29 +237,29 @@ paths: responses: 200: description: |- - The full state for the room, having accepted the join event + The full state for the room, having accepted the join event. schema: type: array minItems: 2 maxItems: 2 items: - type: integer - description: The value ``200`` + description: The value ``200``. example: 200 - type: object title: Room State - description: The state for the room + description: The state for the room. properties: auth_chain: type: array - description: The auth chain + description: The auth chain. items: type: object properties: {} # TODO: Verify schema state: type: array - description: The room state + description: The room state. items: type: object properties: {} From 0d1b82a6017a1e3c0812a3e06d6bd18026a0e96b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 14:40:32 -0600 Subject: [PATCH 224/308] Add the event_id to unsigned PDUs This went missing in the swagger conversion. The example already has an event_id. --- api/server-server/definitions/unsigned_pdu.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml index 011b2736..9e54a837 100644 --- a/api/server-server/definitions/unsigned_pdu.yaml +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -17,6 +17,10 @@ description: An unsigned persistent data unit (event) example: $ref: "../examples/unsigned_pdu.json" properties: + event_id: + type: string + description: The event ID for the PDU + example: "$a4ecee13e2accdadf56c1025:example.com" room_id: type: string description: Room identifier. @@ -107,6 +111,7 @@ properties: description: Additional data added by the origin server but not covered by the ``signatures``. example: {"key": "value"} required: + - event_id - room_id - sender - origin From 9f399a08fe7b7572844704635902e2759405c554 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 14:41:42 -0600 Subject: [PATCH 225/308] Full stops I'm never going to remember these --- api/server-server/definitions/unsigned_pdu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml index 9e54a837..0aeede51 100644 --- a/api/server-server/definitions/unsigned_pdu.yaml +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -19,7 +19,7 @@ example: properties: event_id: type: string - description: The event ID for the PDU + description: The event ID for the PDU. example: "$a4ecee13e2accdadf56c1025:example.com" room_id: type: string From dcc4b0bfbc63f21ce921b4cfbc89133dedbce435 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 14:47:47 -0600 Subject: [PATCH 226/308] Remove Travis CI configuration We're using Circle CI for testing a bunch of the spec, and so far it has the advantage of being faster. Also, it's not named the same as me. We can also remove the Travis CI specific check in generate-matrix-org-assets because Circle CI doesn't touch this. --- .travis.yml | 16 ---------------- scripts/generate-matrix-org-assets | 7 ++----- 2 files changed, 2 insertions(+), 21 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9e9363af..00000000 --- a/.travis.yml +++ /dev/null @@ -1,16 +0,0 @@ -language: go -go: - - 1.8 - -sudo: required - -# we only need a single git commit -git: - depth: 1 - -install: - - sudo apt-get update - - sudo apt-get install python3 python3-dev - -script: - - ./scripts/test-and-build.sh diff --git a/scripts/generate-matrix-org-assets b/scripts/generate-matrix-org-assets index cbebbb64..ed08f81d 100755 --- a/scripts/generate-matrix-org-assets +++ b/scripts/generate-matrix-org-assets @@ -8,11 +8,8 @@ cd `dirname $0`/.. mkdir -p assets -if [ "$TRAVIS" != "true" ] -then - # generate specification/proposals.rst - ./scripts/proposals.py -fi +# generate specification/proposals.rst +./scripts/proposals.py # generate the spec docs ./scripts/gendoc.py -d assets/spec From af0629a5204a5a7188126d97783a6316780399a9 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 08:36:12 -0600 Subject: [PATCH 227/308] Full stop --- api/server-server/joins.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server-server/joins.yaml b/api/server-server/joins.yaml index 11a47af6..eaf14e71 100644 --- a/api/server-server/joins.yaml +++ b/api/server-server/joins.yaml @@ -84,7 +84,7 @@ paths: properties: membership: type: string - description: The value ``join`` + description: The value ``join``. example: "join" required: ['membership'] depth: From 0ddf578b6159e0c8effa5cbbb019efc9290fdff9 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 20 Jul 2018 11:26:16 -0600 Subject: [PATCH 228/308] Combine all queries into their own section This removes the Directory and Profile sections, instead opting to document them as Queries. The behaviour of profile queries is based on Synapse's behaviour. A few issues have been opened to improve the behaviour: * https://github.com/matrix-org/matrix-doc/issues/1434 * https://github.com/matrix-org/matrix-doc/issues/1435 * https://github.com/matrix-org/matrix-doc/issues/1436 * https://github.com/matrix-org/matrix-doc/issues/1437 This fixes https://github.com/matrix-org/matrix-doc/issues/1404 --- api/server-server/directory.yaml | 69 ---------- api/server-server/query.yaml | 181 +++++++++++++++++++++++++++ api/server-server/query_general.yaml | 44 ------- specification/server_server_api.rst | 38 ++---- 4 files changed, 190 insertions(+), 142 deletions(-) delete mode 100644 api/server-server/directory.yaml create mode 100644 api/server-server/query.yaml delete mode 100644 api/server-server/query_general.yaml diff --git a/api/server-server/directory.yaml b/api/server-server/directory.yaml deleted file mode 100644 index 94f115b9..00000000 --- a/api/server-server/directory.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2017 Kamax.io -# Copyright 2018 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -swagger: '2.0' -info: - title: "Matrix Federation Query API" - version: "1.0.0" -host: localhost:8448 -schemes: - - https -basePath: /_matrix/federation/v1 -produces: - - application/json -paths: - "/query/directory": - get: - summary: Retrieve the room ID and list of resident homeservers for a room - alias. - description: Retrieve the room ID and list of resident homeservers for a room - alias. - parameters: - - in: query - name: room_alias - type: string - description: Room alias. - required: true - x-example: "#room_alias:example.org" - responses: - 200: - description: The corresponding room ID and list of known resident - homeservers for the room. - schema: - type: object - properties: - room_id: - type: string - description: The room ID mapped to the queried room alias. - x-example: "!roomid1234:example.org" - servers: - type: array - description: An array of server names that are likely to hold - then given room. This list may or may not include the server - answering the query. - items: - type: string - required: - - "room_id" - - "servers" - examples: - application/json: { - "room_id": "!roomid1234:example.org", - "servers": [ - "example.org", - "example.com", - "another.example.com:8449", - ] - } diff --git a/api/server-server/query.yaml b/api/server-server/query.yaml new file mode 100644 index 00000000..23b9a575 --- /dev/null +++ b/api/server-server/query.yaml @@ -0,0 +1,181 @@ +# Copyright 2017 Kamax.io +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Query API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/query/{queryType}": + get: + summary: Query for information + description: |- + Performs a single query request on the receiving homeserver. The query string + arguments are dependent on which type of query is being made. Known query types + are specified as their own endpoints as an extension to this definition. + operationId: queryInfo + parameters: + - in: path + name: queryType + type: string + description: The type of query to make + required: true + x-example: profile + responses: + 200: + description: |- + The query response. The schema varies depending on the query being made. + "/query/directory": + get: + summary: Query for the room ID and resident homeservers for a room alias + description: |- + Performs a query to get the mapped room ID and list of resident homeservers in + the room for a given room alias. Homeservers should only query room aliases + that belong to the target server (idenfified by the DNS Name in the alias). + The target server may not appear in the resident servers list. + + Servers may wish to cache the response to this query to prevent requesting the + information too often. + operationId: queryRoomDirectory + parameters: + - in: query + name: room_alias + type: string + description: The room alias to query. + required: true + x-example: "#room_alias:example.org" + responses: + 200: + description: |- + The corresponding room ID and list of known resident homeservers for the room. + schema: + type: object + properties: + room_id: + type: string + description: The room ID mapped to the queried room alias. + x-example: "!roomid1234:example.org" + servers: + type: array + description: |- + An array of server names that are likely to hold then given room. This + list may or may not include the server answering the query. + items: + type: string + required: + - "room_id" + - "servers" + examples: + application/json: { + "room_id": "!roomid1234:example.org", + "servers": [ + "example.org", + "example.com", + "another.example.com:8449", + ] + } + 400: + description: |- + The room alias is not hosted on the server. This can happen if the directory + server is named "example.org" and the room alias ends with "matrix.org". + schema: + $ref: "../client-server/definitions/errors/error.yaml" + examples: + application/json: { + "errcode": "M_UNKNOWN", + "error": "Room alias not hosted on this homeserver." + } + 404: + description: The room alias was not found. + schema: + $ref: "../client-server/definitions/errors/error.yaml" + examples: + application/json: { + "errcode": "M_NOT_FOUND", + "error": "Room alias not found." + } + "/query/profile": + get: + summary: Query for profile information about a given user + description: |- + Performs a query to get profile information, such as a display name or avatar, + for a given user. Homeservers should only query profiles for users that belong + to the target server (identified by the DNS Name in the user ID). + + Servers may wish to cache the response to this query to prevent requesting the + information too often. + parameters: + - in: query + name: user_id + type: string + description: The user ID to query. + required: true + x-example: "@someone:example.org" + - in: query + name: field + type: string + description: |- + The field to query. If specified, the server will only return the given field + in the response. If not specified, the server will return the full profile for + the user. + responses: + 200: + description: |- + The profile for the user. If a ``field`` is specified in the request, only the + matching field should be included in the response. If no ``field`` was specified, + the response should include the fields of the user's profile that can be made + public, such as the display name and avatar. + + If the ``field`` is for a field that the server does not recognize, an empty object + should be returned. If the ``field`` is recognized, but the user does not have it + set on their profile, ``null`` should be returned for the value of that field. + + If the user does not exist, an empty object should be returned regardless of the + ``field`` being queried. + schema: + type: object + properties: + displayname: + type: string + description: |- + The display name of the user. If the user does not have a display name set, + this should be the value ``null``. + x-example: "John Doe" + avatar_url: + type: string + description: |- + The avatar URL for the user's avatar. If the user does not have an avatar + set, this should be the value ``null``. + x-example: "mxc://matrix.org/MyC00lAvatar" + examples: + application/json: { + "displayname": "John Doe", + "avatar_url": "mxc://matrix.org/MyC00lAvatar" + } + 400: + description: The user does not belong to the server. + schema: + $ref: "../client-server/definitions/errors/error.yaml" + examples: + application/json: { + "errcode": "M_UNKNOWN", + "error": "User is not hosted on this homeserver." + } diff --git a/api/server-server/query_general.yaml b/api/server-server/query_general.yaml deleted file mode 100644 index 97e3406e..00000000 --- a/api/server-server/query_general.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2018 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -swagger: '2.0' -info: - title: "Matrix Federation Query API" - version: "1.0.0" -host: localhost:8448 -schemes: - - https -basePath: /_matrix/federation/v1 -produces: - - application/json -paths: - "/query/{queryType}": - get: - summary: Query for information - description: |- - Performs a single query request on the receiving homeserver. The query string - arguments are dependent on which type of query is being made. Known query types - are specified as their own endpoints as an extension to this definition. - operationId: queryInfo - parameters: - - in: path - name: queryType - type: string - description: The type of query to make - required: true - x-example: profile - responses: - 200: - description: |- - The query response. The schema varies depending on the query being made. diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f1825f27..b7d5a82f 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -686,8 +686,6 @@ All these URLs are name-spaced within a prefix of:: {{events_ss_http_api}} -{{query_general_ss_http_api}} - {{joins_ss_http_api}} @@ -1107,36 +1105,18 @@ Rejecting a presence invite:: - Explain the zero-byte presence inference logic See also: docs/client-server/model/presence -Profiles --------- +Querying for information +------------------------ -The server API for profiles is based entirely on the following Federation -Queries. There are no additional EDU or PDU types involved, other than the -implicit ``m.presence`` and ``m.room.member`` events (see section below). +Queries are a way to retrieve information from a homeserver abotu a resource, +such as a user or room. The endpoints here are often called in conjunction with +a request from a client on the client-server API in order to complete the call. -Querying profile information:: +There are several types of queries that can be made. The generic endpoint to +represent all queries is described first, followed by the more specific queries +that can be made. - Query type: profile - - Arguments: - user_id: the ID of the user whose profile to return - field: (optional) string giving a field name - - Returns: JSON object containing the following keys: - displayname: string of free-form text - avatar_url: string containing an HTTP-scheme URL - -If the query contains the optional ``field`` key, it should give the name of a -result field. If such is present, then the result should contain only a field -of that name, with no others present. If not, the result should contain as much -of the user's profile as the homeserver has available and can make public. - -Directory ---------- - -The server API for directory queries is also based on Federation Queries. - -{{directory_ss_http_api}} +{{query_ss_http_api}} Send-to-device messaging ------------------------ From d914c402e24e69536902703a305dbf4391c57303 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 16:03:35 -0600 Subject: [PATCH 229/308] Improve the profile query to have more sane responses This is a mix of Synapse and Dendrite behaviour, mostly Dendrite. Synapse returns `null` for field values that aren't set, however Dendrite just doesn't return them and instead opts for an empty object. Further, synapse is lacking in error codes in this area. Dendrite does some additional validation on this API which introduces more errors for bad requests, instead of defaulting to empty objects/200 OK responses. Likewise, Dendrite returns a 404 when the user is not found while Synapse returns 200 OK/empty object. --- api/server-server/query.yaml | 38 ++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/api/server-server/query.yaml b/api/server-server/query.yaml index 23b9a575..4599d130 100644 --- a/api/server-server/query.yaml +++ b/api/server-server/query.yaml @@ -131,7 +131,8 @@ paths: x-example: "@someone:example.org" - in: query name: field - type: string + type: enum + enum: ['displayname', 'avatar_url'] description: |- The field to query. If specified, the server will only return the given field in the response. If not specified, the server will return the full profile for @@ -144,26 +145,22 @@ paths: the response should include the fields of the user's profile that can be made public, such as the display name and avatar. - If the ``field`` is for a field that the server does not recognize, an empty object - should be returned. If the ``field`` is recognized, but the user does not have it - set on their profile, ``null`` should be returned for the value of that field. - - If the user does not exist, an empty object should be returned regardless of the - ``field`` being queried. + If the user does not have a particular field set on their profile, the server + should exclude it from the response body or give it the value ``null``. schema: type: object properties: displayname: type: string description: |- - The display name of the user. If the user does not have a display name set, - this should be the value ``null``. + The display name of the user. May be omitted if the user does not have a + display name set. x-example: "John Doe" avatar_url: type: string description: |- - The avatar URL for the user's avatar. If the user does not have an avatar - set, this should be the value ``null``. + The avatar URL for the user's avatar. May be omitted if the user does not + have an avatar set. x-example: "mxc://matrix.org/MyC00lAvatar" examples: application/json: { @@ -171,11 +168,26 @@ paths: "avatar_url": "mxc://matrix.org/MyC00lAvatar" } 400: - description: The user does not belong to the server. + description: |- + The request was missing parameters or had invalid values for the parameters. This + can happen for: + + * The user not being hosted on the homeserver, + * Lack of a ``user_id`` in the request, or + * The ``field`` requested not being an allowed value. schema: $ref: "../client-server/definitions/errors/error.yaml" examples: application/json: { - "errcode": "M_UNKNOWN", + "errcode": "M_INVALID_ARGUMENT_VALUE", "error": "User is not hosted on this homeserver." } + 404: + description: The user does not exist or does not have a profile. + schema: + $ref: "../client-server/definitions/errors/error.yaml" + examples: + application/json: { + "errcode": "M_NOT_FOUND", + "error": "User does not exist." + } From 8e97b0ca8174190f8bca6170cd39f92d09598d87 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 18 Jul 2018 10:51:28 -0600 Subject: [PATCH 230/308] Improve the server key exchange portion of the s2s specification Most of the text has been shuffled into the swagger definitions to bring it closer to where it matters. This also attempts to clarify what is out in the wild. Most importantly, the first version of the key exchange is outright removed from the specification. Other research points/questions are: * What is a "Key ID"? * https://github.com/matrix-org/synapse/blob/1241156c82644d5609f45659607a356af5d8fe08/synapse/rest/key/v2/local_key_resource.py#L81-L83 * https://github.com/matrix-org/synapse/blob/1241156c82644d5609f45659607a356af5d8fe08/synapse/rest/key/v2/local_key_resource.py#L88-L91 * Returning a cached response if the server throws a 400, 500, or otherwise not-offline status code * https://github.com/matrix-org/synapse/blob/1241156c82644d5609f45659607a356af5d8fe08/synapse/rest/key/v2/remote_key_resource.py#L227-L229 * `minimum_valid_until_ts` default * This branch of the ladder: https://github.com/matrix-org/synapse/blob/1241156c82644d5609f45659607a356af5d8fe08/synapse/rest/key/v2/remote_key_resource.py#L192 * Returning empty arrays when querying offline/no servers * Queried by hand against matrix.org as a notary server with a fake domain name to query * Returning all keys even when querying for specific keys * Queried by hand using matrix.org as a notary server against a server publishing multiple keys. The examples and descriptions were also improved as part of this commit. --- api/server-server/definitions/keys.yaml | 46 ++++--- .../definitions/keys_query_response.yaml | 4 +- api/server-server/examples/server_key.json | 12 +- .../examples/server_key_notary_signed.json | 11 ++ api/server-server/keys_query.yaml | 69 +++++++--- api/server-server/keys_server.yaml | 30 ++++- specification/server_server_api.rst | 120 +++--------------- 7 files changed, 143 insertions(+), 149 deletions(-) create mode 100644 api/server-server/examples/server_key_notary_signed.json diff --git a/api/server-server/definitions/keys.yaml b/api/server-server/definitions/keys.yaml index b08f2465..011ad6fd 100644 --- a/api/server-server/definitions/keys.yaml +++ b/api/server-server/definitions/keys.yaml @@ -20,50 +20,60 @@ properties: server_name: type: string description: DNS name of the homeserver. - required: true # TODO: Verify + required: true example: "example.org" verify_keys: type: object - description: Public keys of the homeserver for verifying digital signatures. - required: true # TODO: Verify + description: |- + Public keys of the homeserver for verifying digital signatures. + + The object's key is the algorithm and version combined (``ed25519`` being the + algorithm and ``abc123`` being the version in the example below). Together, + this forms the Key ID. + required: true additionalProperties: type: object title: Verify Key example: { - "ed25519:auto2": { - "key": "Base+64+Encoded+Signature+Verification+Key" + "ed25519:abc123": { + "key": "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA==" } } properties: key: type: string - description: The key + description: The `Unpadded Base64`_ encoded key. required: true - example: "Base+64+Encoded+Signature+Verification+Key" + example: "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA==" old_verify_keys: type: object - description: The public keys that the server used to use and when it stopped using them. + description: |- + The public keys that the server used to use and when it stopped using them. + + The object's key is the algorithm and version combined (``ed25519`` being the + algorithm and ``0ldK3y`` being the version in the example below). Together, + this forms the Key ID. additionalProperties: type: object title: Old Verify Key example: { - "ed25519:auto1": { + "ed25519:0ldK3y": { "expired_ts": 922834800000, - "key": "Base+64+Encoded+Signature+Verification+Key" + "key": "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg==" } } properties: expired_ts: type: integer format: int64 - description: The expiration time. + description: POSIX timestamp for when this key expired. required: true example: 922834800000 key: type: string - description: The key. + description: The `Unpadded Base64`_ encoded key. required: true - example: "Base+64+Encoded+Signature+Verification+Key" + example: "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg==" signatures: type: object description: Digital signatures for this object signed using the ``verify_keys``. @@ -72,7 +82,7 @@ properties: title: Signed Server example: { "example.org": { - "ad25519:auto2": "Base+64+Encoded+Signature+Verification+Key" + "ad25519:abc123": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU=" } } additionalProperties: @@ -87,10 +97,12 @@ properties: properties: sha256: type: string - description: The encoded fingerprint. - example: Base+64+Encoded+SHA-256-Fingerprint + description: The `Unpadded Base64`_ encoded fingerprint + example: "VGhpcyBpcyBoYXNoIHdoaWNoIHNob3VsZCBiZSBieXRlcw==" valid_until_ts: type: integer format: int64 - description: POSIX timestamp when the list of valid keys should be refreshed. + description: |- + POSIX timestamp when the list of valid keys should be refreshed. Keys used beyond this + timestamp are no longer valid. example: 1052262000000 diff --git a/api/server-server/definitions/keys_query_response.yaml b/api/server-server/definitions/keys_query_response.yaml index bf2d238f..52ad506c 100644 --- a/api/server-server/definitions/keys_query_response.yaml +++ b/api/server-server/definitions/keys_query_response.yaml @@ -15,13 +15,13 @@ type: object description: Server keys example: { "server_keys": [{ - $ref: "../examples/server_key.json" + $ref: "../examples/server_key_notary_signed.json" }] } properties: server_keys: type: array title: Server Keys - description: The server keys. + description: The queried server's keys, signed by the notary server. items: $ref: "keys.yaml" diff --git a/api/server-server/examples/server_key.json b/api/server-server/examples/server_key.json index a3934bd3..8a82d76e 100644 --- a/api/server-server/examples/server_key.json +++ b/api/server-server/examples/server_key.json @@ -1,23 +1,23 @@ { "server_name": "example.org", "verify_keys": { - "ed25519:auto2": { - "key": "Base+64+Encoded+Signature+Verification+Key" + "ed25519:abc123": { + "key": "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA==" } }, "old_verify_keys": { - "ed25519:auto1": { + "ed25519:0ldk3y": { "expired_ts": 922834800000, - "key": "Base+64+Encoded+Old+Verify+Key" + "key": "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg==" } }, "signatures": { "example.org": { - "ed25519:auto2": "Base+64+Encoded+Signature" + "ed25519:auto2": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU=" } }, "tls_fingerprints": [{ - "sha256": "Base+64+Encoded+SHA-256-Fingerprint" + "sha256": "VGhpcyBpcyBoYXNoIHdoaWNoIHNob3VsZCBiZSBieXRlcw==" }], "valid_until_ts": 1052262000000 } \ No newline at end of file diff --git a/api/server-server/examples/server_key_notary_signed.json b/api/server-server/examples/server_key_notary_signed.json new file mode 100644 index 00000000..f83bcc52 --- /dev/null +++ b/api/server-server/examples/server_key_notary_signed.json @@ -0,0 +1,11 @@ +{ + "$ref": "server_key.json", + "signatures": { + "example.org": { + "ed25519:abc123": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU=" + }, + "notary.server.com": { + "ed25519:010203": "VGhpcyBpcyBhbm90aGVyIHNpZ25hdHVyZQ==" + } + } +} \ No newline at end of file diff --git a/api/server-server/keys_query.yaml b/api/server-server/keys_query.yaml index f41cb35b..a1ea80fb 100644 --- a/api/server-server/keys_query.yaml +++ b/api/server-server/keys_query.yaml @@ -25,49 +25,60 @@ produces: paths: "/query/{serverName}/{keyId}": get: - summary: Retrieve a server key. - description: Retrieve a server key. - operationId: perspectivesKeyQuery + summary: Query for another server's keys + description: |- + Query for another server's keys. The receiving (notary) server must + sign the keys returned by the queried server. + operationId: getQueryKeys parameters: - in: path name: serverName type: string - description: Server name. + description: The server's DNS name to query required: true x-example: matrix.org - in: path name: keyId type: string - description: Key ID. - required: true - x-example: TODO # No examples in spec so far + description: |- + The key ID to look up. If omitted or empty, all the server's keys + are to be queried for. + required: false + x-example: "ed25519:abc123" - in: query name: minimum_valid_until_ts type: integer format: int64 - description: Minimum Valid Until Milliseconds. - required: true # TODO: Verify + description: |- + A millisecond POSIX timestamp indicating when the returned certificates + will need to be valid until to be useful to the requesting server. + + If not supplied, the current time as determined by the notary server is used. + required: false x-example: 1234567890 responses: 200: - description: The keys for the server + description: |- + The keys for the server, or an empty array if the server could not be reached + and no cached keys were available. schema: $ref: "definitions/keys_query_response.yaml" "/query": post: - summary: Retrieve a server key - description: Retrieve a server key. - operationId: bulkPerspectivesKeyQuery + summary: Query for several server's keys + description: |- + Query for keys from multiple servers in a batch format. The receiving (notary) + server must sign the keys returned by the queried servers. + operationId: postQueryKeys parameters: - in: body name: body schema: type: object - # TODO: Improve example example: { "server_keys": { - "{server_name}": { - "{key_id}": { + "example.org": { + "ed25519:abc123": { "minimum_valid_until_ts": 1234567890 } } @@ -76,7 +87,16 @@ paths: properties: server_keys: type: object - description: The query criteria. + description: |- + The query criteria. The outer ``string`` key on the object is the + server name (eg: ``matrix.org``). The inner ``string`` key is the + Key ID to query for the particular server. If no key IDs are given + to be queried, the notary server should query for all keys. If no + servers are given, the notary server must return an empty ``server_keys`` + array in the response. + + The notary server may return multiple keys regardless of the Key IDs + given. additionalProperties: type: object name: ServerName @@ -84,16 +104,25 @@ paths: additionalProperties: type: object title: Query Criteria - description: The server keys to query. + description: The server key IDs to query. properties: minimum_valid_until_ts: type: integer format: int64 - description: Minimum Valid Until MS. + description: |- + A millisecond POSIX timestamp indicating when the returned + certificates will need to be valid until to be useful to the + requesting server. + + If not supplied, the current time as determined by the notary + server is used. example: 1234567890 required: ['server_keys'] responses: 200: - description: The keys for the server. + description: |- + The keys for the queried servers, signed by the notary server. Servers which + are offline and have no cached keys will not be included in the result. This + may result in an empty array. schema: $ref: "definitions/keys_query_response.yaml" diff --git a/api/server-server/keys_server.yaml b/api/server-server/keys_server.yaml index 46beeebb..07a2c530 100644 --- a/api/server-server/keys_server.yaml +++ b/api/server-server/keys_server.yaml @@ -25,18 +25,38 @@ produces: paths: "/server/{keyId}": get: - summary: Get the server's key - description: Get the server's key. + summary: Get the homeserver's public key(s) + description: |- + Gets the homeserver's published TLS fingerprints and signing keys. + The homeserver may have any number of active keys and may have a + number of old keys. Homeservers SHOULD return a single JSON object + listing all of its keys, regardless of the ``keyId`` path argument. + This is to reduce the number of round trips needed to discover the + relevant keys for a homeserver. + + Intermediate notary servers should cache a response for half of its + lifetime to avoid serving a stale response. Originating servers should + avoid returning responses that expire in less than an hour to avoid + repeated reqests for a certificate that is about to expire. Requesting + servers should limit how frequently they query for certificates to + avoid flooding a server with requests. + + If the server fails to respond to this request, intermediate notary + servers should continue to return the last response they received + from the server so that the signatures of old events can still be + checked. operationId: getServerKey parameters: - in: path name: keyId type: string - description: Key ID + description: |- + The key ID to look up. If omitted or empty, all server keys are + to be returned. required: false - x-example: TODO # No examples in the spec so far + x-example: "ed25519:abc123" responses: 200: - description: The server's keys. + description: The homeserver's keys schema: $ref: "definitions/keys.yaml" diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f1825f27..4cf92e9b 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -106,15 +106,17 @@ Server implementation Retrieving Server Keys ~~~~~~~~~~~~~~~~~~~~~~ -Version 2 -+++++++++ +.. NOTE:: + There was once a "version 1" of the key exchange. It has been removed from the + specification due to lack of significance. It may be reviewed `here + `_. -Each homeserver publishes its public keys under ``/_matrix/key/v2/server/``. -Homeservers query for keys by either getting ``/_matrix/key/v2/server/`` +Each homeserver publishes its public keys under ``/_matrix/key/v2/server/{keyId}`. +Homeservers query for keys by either getting ``/_matrix/key/v2/server/{keyId}`` directly or by querying an intermediate notary server using a -``/_matrix/key/v2/query`` API. Intermediate notary servers query the -``/_matrix/key/v2/server/`` API on behalf of another server and sign the -response with their own key. A server may query multiple notary servers to +``/_matrix/key/v2/query/{serverName}/{keyId}`` API. Intermediate notary servers +query the ``/_matrix/key/v2/server/{keyId}`` API on behalf of another server and +sign the response with their own key. A server may query multiple notary servers to ensure that they all report the same public keys. This approach is borrowed from the `Perspectives Project`_, but modified to @@ -126,113 +128,33 @@ server by querying other servers. .. _Perspectives Project: https://web.archive.org/web/20170702024706/https://perspectives-project.org/ Publishing Keys -^^^^^^^^^^^^^^^ ++++++++++++++++ Homeservers publish the allowed TLS fingerprints and signing keys in a JSON object at ``/_matrix/key/v2/server/{key_id}``. The response contains a list of ``verify_keys`` that are valid for signing federation requests made by the -server and for signing events. It contains a list of ``old_verify_keys`` which +homeserver and for signing events. It contains a list of ``old_verify_keys`` which are only valid for signing events. Finally the response contains a list of TLS -certificate fingerprints to validate any connection made to the server. - -A server may have multiple keys active at a given time. A server may have any -number of old keys. It is recommended that servers return a single JSON -response listing all of its keys whenever any ``key_id`` is requested to reduce -the number of round trips needed to discover the relevant keys for a server. -However a server may return different responses for a different ``key_id``. - -The ``tls_certificates`` field contains a list of hashes of the X.509 TLS -certificates currently used by the server. The list must include SHA-256 hashes -for every certificate currently in use by the server. These fingerprints are -valid until the millisecond POSIX timestamp in ``valid_until_ts``. - -The ``verify_keys`` can be used to sign requests and events made by the server -until the millisecond POSIX timestamp in ``valid_until_ts``. If a homeserver -receives an event with a ``origin_server_ts`` after the ``valid_until_ts`` then -it should request that ``key_id`` for the originating server to check whether -the key has expired. - -The ``old_verify_keys`` can be used to sign events with an ``origin_server_ts`` -before the ``expired_ts``. The ``expired_ts`` is a millisecond POSIX timestamp -of when the originating server stopped using that key. - -Intermediate notary servers should cache a response for half of its remaining -lifetime to avoid serving a stale response. Originating servers should avoid -returning responses that expire in less than an hour to avoid repeated requests -for a certificate that is about to expire. Requesting servers should limit how -frequently they query for certificates to avoid flooding a server with -requests. - -If a server goes offline intermediate notary servers should continue to return -the last response they received from that server so that the signatures of old -events sent by that server can still be checked. +certificate fingerprints to validate any connection made to the homeserver. {{keys_server_ss_http_api}} Querying Keys Through Another Server -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +++++++++++++++++++++++++++++++++++++ -Servers may offer a query API ``/_matrix/key/v2/query/`` for getting the keys -for another server. This API can be used to GET a list of JSON objects for a -given server or to POST a bulk query for a number of keys from a number of -servers. Either way the response is a list of JSON objects containing the -JSON published by the server under ``/_matrix/key/v2/server/`` signed by -both the originating server and by this server. +Servers may query another server's keys through a notary server. The notary +server may be another homeserver. The notary server will retrieve keys from +the queried servers through use of the ``/_matrix/key/v2/server/{keyId}`` +API. The notary server will additionally sign the response from the queried +server before returning the results. -The ``minimum_valid_until_ts`` is a millisecond POSIX timestamp indicating -when the returned certificate will need to be valid until to be useful to the -requesting server. This can be set using the maximum ``origin_server_ts`` of -a batch of events that a requesting server is trying to validate. This allows -an intermediate notary server to give a prompt cached response even if the -originating server is offline. - -This API can return keys for servers that are offline by using cached responses -taken from when the server was online. Keys can be queried from multiple -servers to mitigate against DNS spoofing. +Notary servers can return keys for servers that are offline or having issues +serving their own keys by using cached responses. Keys can be queried from +multiple servers to mitigate against DNS spoofing. {{keys_query_ss_http_api}} -Version 1 -+++++++++ -.. WARNING:: - Version 1 of key distribution is obsolete. - - -Homeservers publish their TLS certificates and signing keys in a JSON object -at ``/_matrix/key/v1``. - -==================== =================== ====================================== - Key Type Description -==================== =================== ====================================== -``server_name`` String DNS name of the homeserver. -``verify_keys`` Object Public keys of the homeserver for - verifying digital signatures. -``signatures`` Object Digital signatures for this object - signed using the ``verify_keys``. -``tls_certificate`` String The X.509 TLS certificate used by this - this server encoded as `Unpadded Base64`_. -==================== =================== ====================================== - -.. code:: json - - { - "server_name": "example.org", - "signatures": { - "example.org": { - "ed25519:auto": "Base+64+Encoded+Signature" - } - }, - "tls_certificate": "Base+64+Encoded+DER+Encoded+X509+TLS+Certificate", - "verify_keys": { - "ed25519:auto": "Base+64+Encoded+Signature+Verification+Key" - } - } - -When fetching the keys for a server the client should check that the TLS -certificate in the JSON matches the TLS server certificate for the connection -and should check that the JSON signatures are correct for the supplied -``verify_keys``. Transactions ------------ From c2f1c6e78d4d92aa77e9df131dfc9234a5a3e614 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 18 Jul 2018 16:21:48 -0600 Subject: [PATCH 231/308] Improve the joining rooms handshake documentation There isn't a whole lot to this section that needed work. The section overall lost the table schema in favour of having the endpoints close by. The directory query is improved in https://github.com/matrix-org/matrix-doc/pull/1443 --- api/server-server/joins.yaml | 29 +++++--- specification/server_server_api.rst | 100 ++++++---------------------- 2 files changed, 39 insertions(+), 90 deletions(-) diff --git a/api/server-server/joins.yaml b/api/server-server/joins.yaml index eaf14e71..472d63bf 100644 --- a/api/server-server/joins.yaml +++ b/api/server-server/joins.yaml @@ -29,7 +29,6 @@ paths: description: |- Asks the receiving server to return information that the sending server will need to prepare a join event to get into the room. - This is part of the `Joining Rooms`_ handshake. operationId: makeJoin parameters: - in: path @@ -95,7 +94,9 @@ paths: type: array description: |- An event reference list containing the authorization events that would - allow the member to join the room. + allow the member to join the room. This should normally be the + ``m.room.create``, ``m.room.power_levels``, and ``m.room.join_rules`` + events. items: type: array maxItems: 2 @@ -128,7 +129,12 @@ paths: "state_key": "@someone:example.org", "content": { "membership": "join" - } + }, + "auth_events": [ + ["$room_cre4te_3vent:matrix.org", {"sha256": "abase64encodedsha256hashshouldbe43byteslong"}], + ["$room_j0in_rul3s_3vent:matrix.org", {"sha256": "abase64encodedsha256hashshouldbe43byteslong"}], + ["$room_p0wer_l3vels_3vent:matrix.org", {"sha256": "abase64encodedsha256hashshouldbe43byteslong"}] + ] } "/send_join/{roomId}/{eventId}": put: @@ -250,27 +256,30 @@ paths: title: Room State description: The state for the room. properties: + origin: + type: string + description: The resident server's DNS name. auth_chain: type: array description: The auth chain. items: type: object - properties: {} - # TODO: Verify schema + schema: + $ref: "definitions/pdu.yaml" state: type: array description: The room state. items: type: object - properties: {} - # TODO: Verify schema + schema: + $ref: "definitions/pdu.yaml" required: ["auth_chain", "state"] examples: application/json: [ 200, { - # TODO: Use the appropriate refs (see TODOs in schema) - "auth_chain": [], - "state": [] + "origin": "matrix.org", + "auth_chain": [{"$ref": "examples/pdu.json"}], + "state": [{"$ref": "examples/pdu.json"}] } ] diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f1825f27..d819b072 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -688,9 +688,6 @@ All these URLs are name-spaced within a prefix of:: {{query_general_ss_http_api}} - -{{joins_ss_http_api}} - Joining Rooms ------------- @@ -742,94 +739,34 @@ homeservers, though most in practice will use just two. <---------- join response The first part of the handshake usually involves using the directory server to -request the room ID and join candidates. This is covered in more detail on the -directory server documentation, below. In the case of a new user joining a -room as a result of a received invite, the joining user's homeserver could -optimise this step away by picking the origin server of that invite message as -the join candidate. However, the joining server should be aware that the origin -server of the invite might since have left the room, so should be prepared to -fall back on the regular join flow if this optimisation fails. +request the room ID and join candidates through the |/query/directory|_ +API endpoint. In the case of a new user joining a room as a result of a received +invite, the joining user's homeserver could optimise this step away by picking +the origin server of that invite message as the join candidate. However, the +joining server should be aware that the origin server of the invite might since +have left the room, so should be prepared to fall back on the regular join flow +if this optimisation fails. Once the joining server has the room ID and the join candidates, it then needs to obtain enough information about the room to fill in the required fields of the ``m.room.member`` event. It obtains this by selecting a resident from the -candidate list, and requesting the ``make_join`` endpoint using a ``GET`` -request, specifying the room ID and the user ID of the new member who is -attempting to join. +candidate list, and using the ``GET /make_join`` endpoint. The resident server +will then reply with enough information for the joining server to fill in the +event. -The resident server replies to this request with a JSON-encoded object having a -single key called ``event``; within this is an object whose fields contain some -of the information that the joining server will need. Despite its name, this -object is not a full event; notably it does not need to be hashed or signed by -the resident homeserver. The required fields are: - -======================== ============ ========================================= - Key Type Description -======================== ============ ========================================= -``type`` String The value ``m.room.member``. -``auth_events`` List An event-reference list containing the - authorization events that would allow - this member to join. -``content`` Object The event content. -``depth`` Integer (this field must be present but is - ignored; it may be 0) -``origin`` String The name of the resident homeserver. -``origin_server_ts`` Integer A timestamp added by the resident - homeserver. -``prev_events`` List An event-reference list containing the - immediate predecessor events. -``room_id`` String The room ID of the room. -``sender`` String The user ID of the joining member. -``state_key`` String The user ID of the joining member. -======================== ============ ========================================= - -The ``content`` field itself must be an object, containing: - -======================== ============ ========================================= - Key Type Description -======================== ============ ========================================= -``membership`` String The value ``join``. -======================== ============ ========================================= - -The joining server now has sufficient information to construct the real join -event from these protoevent fields. It copies the values of most of them, -adding (or replacing) the following fields: - -======================== ============ ========================================= - Key Type Description -======================== ============ ========================================= -``event_id`` String A new event ID specified by the joining - homeserver. -``origin`` String The name of the joining homeserver. -``origin_server_ts`` Integer A timestamp added by the joining - homeserver. -======================== ============ ========================================= - -This will be a true event, so the joining server should apply the event-signing -algorithm to it, resulting in the addition of the ``hashes`` and ``signatures`` -fields. +The joining server is expected to add or replace the ``origin``, ``origin_server_ts``, +and ``event_id`` on the templated event received by the resident server. This +event is then signed by the joining server. To complete the join handshake, the joining server must now submit this new -event to an resident homeserver, by using the ``send_join`` endpoint. This is -invoked using the room ID and the event ID of the new member event. +event to an resident homeserver, by using the ``PUT /send_join`` endpoint. The resident homeserver then accepts this event into the room's event graph, and responds to the joining server with the full set of state for the -newly-joined room. This is returned as a two-element list, whose first element -is the integer 200, and whose second element is an object which contains the -following keys: +newly-joined room. The resident server must also send the event to other servers +participating in the room. -======================== ============ ========================================= - Key Type Description -======================== ============ ========================================= -``auth_chain`` List A list of events giving all of the events - in the auth chains for the join event and - the events in ``state``. -``state`` List A complete list of the prevailing state - events at the instant just before - accepting the new ``m.room.member`` - event. -======================== ============ ========================================= +{{joins_ss_http_api}} .. TODO-spec - (paul) I don't really understand why the full auth_chain events are given @@ -1306,6 +1243,9 @@ that are too long. [[TODO(markjh) We might want to allow the server to omit the output of well known hash functions like SHA-256 when none of the keys have been redacted]] +.. |/query/directory| replace:: ``/query/directory`` +.. _/query/directory: #get-matrix-federation-v1-query-directory + .. _`Invitation storage`: ../identity_service/unstable.html#invitation-storage .. _`Identity Service API`: ../identity_service/unstable.html .. _`Client-Server API`: ../client_server/unstable.html#m-room-member From f873bae0cc0339cd9b275d755b1534772f8757a1 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 24 Jul 2018 16:46:10 -0600 Subject: [PATCH 232/308] Improve documentation for how non-third party invites work The details are fairly straightforward. An `event` has been added to the response body because that's what Synapse returns, despite the spec saying otherwise until now: https://github.com/matrix-org/synapse/blob/d69decd5c78c72abef50b597a689e2bc55a39702/synapse/federation/federation_server.py#L339 --- .../definitions/invite_event.yaml | 3 +- api/server-server/definitions/pdu.yaml | 2 +- api/server-server/invites.yaml | 63 ++++++++++++++----- specification/server_server_api.rst | 5 ++ 4 files changed, 55 insertions(+), 18 deletions(-) diff --git a/api/server-server/definitions/invite_event.yaml b/api/server-server/definitions/invite_event.yaml index 9cef8061..d196339a 100644 --- a/api/server-server/definitions/invite_event.yaml +++ b/api/server-server/definitions/invite_event.yaml @@ -47,7 +47,7 @@ allOf: title: Membership Event Content description: |- The content of the event, matching what is available in the - `Client-Server API`_. + `Client-Server API`_. Must include a ``membership`` of ``invite``. example: {"membership": "invite"} properties: membership: @@ -85,4 +85,3 @@ allOf: required: # Every other field is already flagged as required by the $ref - state_key - - unsigned # TODO: apparently this is required? Verify. diff --git a/api/server-server/definitions/pdu.yaml b/api/server-server/definitions/pdu.yaml index a5da57c0..bb14ede2 100644 --- a/api/server-server/definitions/pdu.yaml +++ b/api/server-server/definitions/pdu.yaml @@ -36,7 +36,7 @@ allOf: signatures: type: object description: |- - Signatures of the redacted PDU, following the algorithm specified in `Signing Events`_. + Signatures for the PDU, following the algorithm specified in `Signing Events`_. example: { "example.com": { "ed25519:key_version:": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus" diff --git a/api/server-server/invites.yaml b/api/server-server/invites.yaml index a5d21938..4a0a610e 100644 --- a/api/server-server/invites.yaml +++ b/api/server-server/invites.yaml @@ -25,11 +25,11 @@ produces: paths: "/invite/{roomId}/{eventId}": put: - summary: Invites a user to a room + summary: Invites a remote user to a room description: |- - Invites a remote user to a room. Once the event has been - signed by both the inviting homeserver and the invited - homeserver, it can be sent to all of the users in the room. + Invites a remote user to a room. Once the event has been signed by both the inviting + homeserver and the invited homeserver, it can be sent to all of the servers in the + room by the inviting homeserver. operationId: sendInvite parameters: - in: path @@ -41,7 +41,7 @@ paths: - in: path name: eventId type: string - description: The event ID for the invite event. + description: The event ID for the invite event, generated by the inviting server. required: true x-example: "$abc123:example.org" - in: body @@ -53,11 +53,15 @@ paths: example: { "$ref": "examples/pdu.json", "type": "m.room.member", - "state_key": "@someone:example.org", + "state_key": "@joe:elsewhere.com", "content": { "membership": "invite" }, - "unsigned": {} + "signatures": { + "example.com": { + "ed25519:key_version": "SomeSignatureHere" + }, + } } responses: 200: @@ -72,17 +76,46 @@ paths: - type: integer description: The value ``200``. example: 200 - - $ref: "definitions/invite_event.yaml" + - type: object + description: An object containing the signed invite event. + title: Event Container + properties: + event: + $ref: "definitions/invite_event.yaml" + required: ['event'] examples: application/json: [ 200, { - "$ref": "examples/pdu.json", - "type": "m.room.member", - "state_key": "@someone:example.org", - "content": { - "membership": "invite" - }, - "unsigned": {} + "event": { + "$ref": "examples/pdu.json", + "type": "m.room.member", + "state_key": "@someone:example.org", + "content": { + "membership": "invite" + }, + "signatures": { + "example.com": { + "ed25519:key_version": "SomeSignatureHere" + }, + "elsewhere.com": { + "ed25519:k3y_versi0n": "SomeOtherSignatureHere" + } + } + } } ] + 403: + description: |- + The invite is not allowed. This could be for a number of reasons, including: + + * The sender is not allowed to send invites to the target user/homeserver. + * The homeserver does not permit anyone to invite its users. + * The homeserver refuses to participate in the room. + schema: + $ref: "../client-server/definitions/errors/error.yaml" + examples: + application/json: { + "errcode": "M_FORBIDDEN", + "error": "User cannot invite the target user." + } diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f1825f27..9d48f113 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -879,6 +879,11 @@ that requested by the requester in the ``v`` parameter). Inviting to a room ------------------ +When a user on a given homeserver invites another user on the same homeserver, +the homeserver may sign the membership event itself and skip the process defined +here. However, when a user invites another user on a different homeserver, a request +to that homeserver to have the event signed and verified must be made. + {{invites_ss_http_api}} Third-party invites From fcaa299f8111137ce932ea971ad801b37bce1dc6 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 25 Jul 2018 11:58:13 -0600 Subject: [PATCH 233/308] Include information on how invite_room_state works This is completely optional, and the events included are up to the sender. Sources: * Structure: https://github.com/matrix-org/synapse/blob/d69decd5c78c72abef50b597a689e2bc55a39702/synapse/handlers/message.py#L896-L904 * What events are recommended: https://github.com/matrix-org/synapse/blob/d69decd5c78c72abef50b597a689e2bc55a39702/synapse/config/api.py#L30-L40 --- api/server-server/invites.yaml | 88 +++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/api/server-server/invites.yaml b/api/server-server/invites.yaml index 4a0a610e..d1838bf5 100644 --- a/api/server-server/invites.yaml +++ b/api/server-server/invites.yaml @@ -49,11 +49,77 @@ paths: type: object required: true schema: - $ref: "definitions/invite_event.yaml" + allOf: + - $ref: "definitions/invite_event.yaml" + - type: object + properties: + unsigned: + type: object + title: Unsigned Event Content + description: |- + Information included alongside the event that is not signed. May include more + than what is listed here. + properties: + invite_room_state: + type: array + description: |- + An optional list of simplified events to help the receiver of the invite + identify the room. The recommended events to include are the join rules, + canonical alias, avatar, and name of the room. + items: + type: object + title: Invite Room State Event + properties: + type: + type: string + description: The type of event. + example: "m.room.join_rules" + state_key: + type: string + description: The state key for the event. May be an empty string. + example: "" + content: + type: object + description: The content for the event. + sender: + type: string + description: The sender of the event. + example: "@someone:matrix.org" + required: ['type', 'state_key', 'content', 'sender'] + example: [ + { + "type": "m.room.join_rules", + "sender": "@someone:matrix.org", + "state_key": "", + "content": { + "join_rule": "public" + } + } + ] example: { "$ref": "examples/pdu.json", "type": "m.room.member", "state_key": "@joe:elsewhere.com", + "unsigned": { + "invite_room_state": [ + { + "type": "m.room.join_rules", + "sender": "@someone:matrix.org", + "state_key": "", + "content": { + "join_rule": "public" + } + }, + { + "type": "m.room.name", + "sender": "@someone:matrix.org", + "state_key": "", + "content": { + "name": "Cool New Room" + } + } + ] + }, "content": { "membership": "invite" }, @@ -91,6 +157,26 @@ paths: "$ref": "examples/pdu.json", "type": "m.room.member", "state_key": "@someone:example.org", + "unsigned": { + "invite_room_state": [ + { + "type": "m.room.join_rules", + "sender": "@someone:matrix.org", + "state_key": "", + "content": { + "join_rule": "public" + } + }, + { + "type": "m.room.name", + "sender": "@someone:matrix.org", + "state_key": "", + "content": { + "name": "Cool New Room" + } + } + ] + }, "content": { "membership": "invite" }, From 4f24d2101f5d84f5c69ad9665b94a3569facc831 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 09:51:08 -0600 Subject: [PATCH 234/308] Only get issues that are proposals when finding proposals Otherwise we end up with not-proposals showing up in the list. --- scripts/proposals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index eec6d652..867d35d5 100755 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -28,7 +28,7 @@ def getpage(url, page): def getbylabel(label): pagecount = 1 json = list() - urlbase = 'https://api.github.com/repos/matrix-org/matrix-doc/issues?state=all&labels=' + label + '&page=' + urlbase = 'https://api.github.com/repos/matrix-org/matrix-doc/issues?state=all&labels=proposal,' + label + '&page=' print(urlbase) json.extend(getpage(urlbase, 1)) for page in range(2, int(pagecount) + 1): From 9fdd8a6f964c70689bd7451d746e96164ce8ab5f Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 14:59:43 -0600 Subject: [PATCH 235/308] Document how leaving rooms/rejecting invites over federation works Fixes https://github.com/matrix-org/matrix-doc/issues/1401 This is very similar to the joining rooms handshake, and much of it is a near copy/paste of the make_join and send_join API. The major difference is the send_leave API doesn't return anything. References: * Handling of make_leave: https://github.com/matrix-org/synapse/blob/d69decd5c78c72abef50b597a689e2bc55a39702/synapse/handlers/federation.py#L1285-L1310 * send_leave route: https://github.com/matrix-org/synapse/blob/d69decd5c78c72abef50b597a689e2bc55a39702/synapse/federation/transport/client.py#L267 * make_leave route: https://github.com/matrix-org/synapse/blob/d69decd5c78c72abef50b597a689e2bc55a39702/synapse/federation/transport/server.py#L396 * send_leave returning nothing: https://github.com/matrix-org/synapse/blob/d69decd5c78c72abef50b597a689e2bc55a39702/synapse/handlers/federation.py#L1346 --- api/server-server/leaving.yaml | 266 ++++++++++++++++++++++++++++ specification/server_server_api.rst | 20 +++ 2 files changed, 286 insertions(+) create mode 100644 api/server-server/leaving.yaml diff --git a/api/server-server/leaving.yaml b/api/server-server/leaving.yaml new file mode 100644 index 00000000..11b0e01d --- /dev/null +++ b/api/server-server/leaving.yaml @@ -0,0 +1,266 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Leave Room API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/make_leave/{roomId}/{userId}": + get: + summary: Get information required to make a join event for a room + description: |- + Asks the receiving server to return information that the sending + server will need to prepare a leave event to get out of the room. + operationId: makeLeave + parameters: + - in: path + name: roomId + type: string + description: The room ID that is about to be left. + required: true + x-example: "!abc123:matrix.org" + - in: path + name: userId + type: string + description: The user ID the leave event will be for. + required: true + x-example: "@someone:example.org" + responses: + 200: + description: |- + An unsigned event that the sending server may use as a template + for when it calls ``/send_leave``. + schema: + allOf: + - $ref: "definitions/unsigned_pdu.yaml" + - type: object + properties: + # Note: we override a bunch of parameters to change their descriptions + sender: + type: string + description: The user ID of the leaving member. + example: "@someone:example.org" + origin: + type: string + description: The name of the resident homeserver. + example: "matrix.org" + origin_server_ts: + type: integer + format: int64 + description: A timestamp added by the resident homeserver. + example: 1234567890 + type: + type: string + description: The value ``m.room.member``. + example: "m.room.member" + state_key: + type: string + description: The user ID of the leaving member. + example: "@someone:example.org" + content: + type: object + title: Membership Event Content + description: The content of the event. + example: {"membership": "leave"} + properties: + membership: + type: string + description: The value ``leave``. + example: "leave" + required: ['membership'] + auth_events: + type: array + description: |- + An event reference list containing the authorization events that would + allow the member to leave the room. This should normally be the + ``m.room.create``, ``m.room.power_levels``, and ``m.room.join_rules`` + events. + items: + type: array + maxItems: 2 + minItems: 2 + items: + - type: string + title: Event ID + example: "$abc123:matrix.org" + - type: object + title: Event Hash + example: { + "sha256": "abase64encodedsha256hashshouldbe43byteslong" + } + properties: + sha256: + type: string + description: The event hash. + example: abase64encodedsha256hashshouldbe43byteslong + required: ['sha256'] + redacts: + type: string + description: Not used. + required: + # Every other field is already flagged as required by the $ref + - state_key + examples: + application/json: { + "$ref": "examples/unsigned_pdu.json", + "type": "m.room.member", + "state_key": "@someone:example.org", + "content": { + "membership": "leave" + }, + "auth_events": [ + ["$room_cre4te_3vent:matrix.org", {"sha256": "abase64encodedsha256hashshouldbe43byteslong"}], + ["$room_j0in_rul3s_3vent:matrix.org", {"sha256": "abase64encodedsha256hashshouldbe43byteslong"}], + ["$room_p0wer_l3vels_3vent:matrix.org", {"sha256": "abase64encodedsha256hashshouldbe43byteslong"}] + ] + } + 403: + description: |- + The request is not authorized. This could mean that the user is not in the room. + schema: + $ref: "../client-server/definitions/errors/error.yaml" + examples: + application/json: { + "errcode": "M_FORBIDDEN", + "error": "User is not in the room." + } + "/send_leave/{roomId}/{eventId}": + put: + summary: Submit a signed leave event to a resident server + description: |- + Submits a signed leave event to the resident server for it + to accept it into the room's graph. + operationId: sendLeave + parameters: + - in: path + name: roomId + type: string + description: The room ID that is about to be left. + required: true + x-example: "!abc123:matrix.org" + - in: path + name: eventId + type: string + description: The event ID for the leave event. + required: true + x-example: "$abc123:example.org" + - in: body + name: body + type: object + required: true + schema: + allOf: + - $ref: "definitions/pdu.yaml" + - type: object + properties: + # Note: we override a bunch of parameters to change their descriptions + sender: + type: string + description: The user ID of the leaving member. + example: "@someone:example.org" + origin: + type: string + description: The name of the leaving homeserver. + example: "matrix.org" + origin_server_ts: + type: integer + format: int64 + description: A timestamp added by the leaving homeserver. + example: 1234567890 + type: + type: string + description: The value ``m.room.member``. + example: "m.room.member" + state_key: + type: string + description: The user ID of the leaving member. + example: "@someone:example.org" + content: + type: object + title: Membership Event Content + description: The content of the event. + example: {"membership": "leave"} + properties: + membership: + type: string + description: The value ``leave``. + example: "leave" + required: ['membership'] + depth: + type: integer + description: This field must be present but is ignored; it may be 0. + example: 12 + auth_events: + type: array + description: |- + An event reference list containing the authorization events that would + allow the member to leave the room. + items: + type: array + maxItems: 2 + minItems: 2 + items: + - type: string + title: Event ID + example: "$abc123:matrix.org" + - type: object + title: Event Hash + example: { + "sha256": "abase64encodedsha256hashshouldbe43byteslong" + } + properties: + sha256: + type: string + description: The event hash. + example: abase64encodedsha256hashshouldbe43byteslong + required: ['sha256'] + redacts: + type: string + description: Not used. + required: + # Every other field is already flagged as required by the $ref + - state_key + example: { + "$ref": "examples/pdu.json", + "type": "m.room.member", + "state_key": "@someone:example.org", + "content": { + "membership": "leave" + } + } + responses: + 200: + description: |- + An empty response to indicate the event was accepted into the graph by + the receiving homeserver. + schema: + type: array + minItems: 2 + maxItems: 2 + items: + - type: integer + description: The value ``200``. + example: 200 + - type: object + title: Empty Object + description: An empty object. + examples: + application/json: [200, {}] diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f1825f27..32b6486f 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -881,6 +881,26 @@ Inviting to a room {{invites_ss_http_api}} +Leaving Rooms (Rejecting Invites) +--------------------------------- + +Normally homeservers can send appropriate ``m.room.member`` events to have users +leave the room, or to reject local invites. Remote invites from other homeservers +do not involve the server in the graph and therefore need another approach to +reject the invite. Joining the room and promptly leaving is not recommended as +clients and servers will interpret that as accepting the invite, then leaving the +room rather than rejecting the invite. + +Similar to the `Joining Rooms`_ handshake, the server which wishes to leave the +room starts with sending a ``/make_leave`` request to a resident server. In the +case of rejecting invites, the resident server may be the server which sent the +invite. After receiving a template event from ``/make_leave``, the leaving server +signs the event and replaces the ``event_id`` with it's own. This is then sent to +the resident server via ``/send_leave``. The resident server will then send the +event to other servers in the room. + +{{leaving_ss_http_api}} + Third-party invites ------------------- From 2f71703427f87ea2d612025b50b37709202b639e Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 16:42:21 -0600 Subject: [PATCH 236/308] Deprecate key ID querying --- api/server-server/keys_query.yaml | 5 +++-- api/server-server/keys_server.yaml | 11 +++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/server-server/keys_query.yaml b/api/server-server/keys_query.yaml index a1ea80fb..c2800bc6 100644 --- a/api/server-server/keys_query.yaml +++ b/api/server-server/keys_query.yaml @@ -41,8 +41,9 @@ paths: name: keyId type: string description: |- - The key ID to look up. If omitted or empty, all the server's keys - are to be queried for. + **Deprecated**. Servers should not use this parameter and instead + opt to return all keys, not just the requested one. The key ID to + look up. required: false x-example: "ed25519:abc123" - in: query diff --git a/api/server-server/keys_server.yaml b/api/server-server/keys_server.yaml index 07a2c530..8734f2ed 100644 --- a/api/server-server/keys_server.yaml +++ b/api/server-server/keys_server.yaml @@ -29,10 +29,7 @@ paths: description: |- Gets the homeserver's published TLS fingerprints and signing keys. The homeserver may have any number of active keys and may have a - number of old keys. Homeservers SHOULD return a single JSON object - listing all of its keys, regardless of the ``keyId`` path argument. - This is to reduce the number of round trips needed to discover the - relevant keys for a homeserver. + number of old keys. Intermediate notary servers should cache a response for half of its lifetime to avoid serving a stale response. Originating servers should @@ -51,10 +48,12 @@ paths: name: keyId type: string description: |- - The key ID to look up. If omitted or empty, all server keys are - to be returned. + **Deprecated**. Servers should not use this parameter and instead + opt to return all keys, not just the requested one. The key ID to + look up. required: false x-example: "ed25519:abc123" + deprecated: true responses: 200: description: The homeserver's keys From 9dbb627d7b91ec795bf3d2ce5e555af65b6c180a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 16:42:40 -0600 Subject: [PATCH 237/308] Undo accidental changes to the operation ID --- api/server-server/keys_query.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/server-server/keys_query.yaml b/api/server-server/keys_query.yaml index c2800bc6..f6e2cb19 100644 --- a/api/server-server/keys_query.yaml +++ b/api/server-server/keys_query.yaml @@ -29,7 +29,7 @@ paths: description: |- Query for another server's keys. The receiving (notary) server must sign the keys returned by the queried server. - operationId: getQueryKeys + operationId: perspectivesKeyQuery parameters: - in: path name: serverName @@ -70,7 +70,7 @@ paths: description: |- Query for keys from multiple servers in a batch format. The receiving (notary) server must sign the keys returned by the queried servers. - operationId: postQueryKeys + operationId: bulkPerspectivesKeyQuery parameters: - in: body name: body From cad1db2a14aa7592b37a8225fe5bd307691d9700 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 16:43:16 -0600 Subject: [PATCH 238/308] Unpadded means unpadded --- api/server-server/definitions/keys.yaml | 16 ++++++++-------- .../examples/server_key_notary_signed.json | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/api/server-server/definitions/keys.yaml b/api/server-server/definitions/keys.yaml index 011ad6fd..2f154236 100644 --- a/api/server-server/definitions/keys.yaml +++ b/api/server-server/definitions/keys.yaml @@ -36,7 +36,7 @@ properties: title: Verify Key example: { "ed25519:abc123": { - "key": "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA==" + "key": "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA" } } properties: @@ -44,7 +44,7 @@ properties: type: string description: The `Unpadded Base64`_ encoded key. required: true - example: "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA==" + example: "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA" old_verify_keys: type: object description: |- @@ -59,7 +59,7 @@ properties: example: { "ed25519:0ldK3y": { "expired_ts": 922834800000, - "key": "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg==" + "key": "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg" } } properties: @@ -73,7 +73,7 @@ properties: type: string description: The `Unpadded Base64`_ encoded key. required: true - example: "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg==" + example: "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg" signatures: type: object description: Digital signatures for this object signed using the ``verify_keys``. @@ -82,7 +82,7 @@ properties: title: Signed Server example: { "example.org": { - "ad25519:abc123": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU=" + "ad25519:abc123": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU" } } additionalProperties: @@ -90,15 +90,15 @@ properties: name: Encoded Signature Verification Key tls_fingerprints: type: array - description: Hashes of X.509 TLS certificates used by this server encoded as `Unpadded Base64`_. + description: Hashes of X.509 TLS certificates used by this server. items: type: object title: TLS Fingerprint properties: sha256: type: string - description: The `Unpadded Base64`_ encoded fingerprint - example: "VGhpcyBpcyBoYXNoIHdoaWNoIHNob3VsZCBiZSBieXRlcw==" + description: The `Unpadded Base64`_ encoded fingerprint. + example: "VGhpcyBpcyBoYXNoIHdoaWNoIHNob3VsZCBiZSBieXRlcw" valid_until_ts: type: integer format: int64 diff --git a/api/server-server/examples/server_key_notary_signed.json b/api/server-server/examples/server_key_notary_signed.json index f83bcc52..d3a461ba 100644 --- a/api/server-server/examples/server_key_notary_signed.json +++ b/api/server-server/examples/server_key_notary_signed.json @@ -2,10 +2,10 @@ "$ref": "server_key.json", "signatures": { "example.org": { - "ed25519:abc123": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU=" + "ed25519:abc123": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU" }, "notary.server.com": { - "ed25519:010203": "VGhpcyBpcyBhbm90aGVyIHNpZ25hdHVyZQ==" + "ed25519:010203": "VGhpcyBpcyBhbm90aGVyIHNpZ25hdHVyZQ" } } } \ No newline at end of file From bdccfca72679a22bc67b5ad522ed7e324067f5e3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 16:43:43 -0600 Subject: [PATCH 239/308] Timestamps should be in milliseconds --- api/server-server/definitions/keys.yaml | 2 +- api/server-server/keys_query.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/server-server/definitions/keys.yaml b/api/server-server/definitions/keys.yaml index 2f154236..67908268 100644 --- a/api/server-server/definitions/keys.yaml +++ b/api/server-server/definitions/keys.yaml @@ -66,7 +66,7 @@ properties: expired_ts: type: integer format: int64 - description: POSIX timestamp for when this key expired. + description: POSIX timestamp in milliseconds for when this key expired. required: true example: 922834800000 key: diff --git a/api/server-server/keys_query.yaml b/api/server-server/keys_query.yaml index f6e2cb19..8fbe00dc 100644 --- a/api/server-server/keys_query.yaml +++ b/api/server-server/keys_query.yaml @@ -51,8 +51,8 @@ paths: type: integer format: int64 description: |- - A millisecond POSIX timestamp indicating when the returned certificates - will need to be valid until to be useful to the requesting server. + A millisecond POSIX timestamp in milliseconds indicating when the returned + certificates will need to be valid until to be useful to the requesting server. If not supplied, the current time as determined by the notary server is used. required: false @@ -111,9 +111,9 @@ paths: type: integer format: int64 description: |- - A millisecond POSIX timestamp indicating when the returned - certificates will need to be valid until to be useful to the - requesting server. + A millisecond POSIX timestamp in milliseconds indicating when + the returned certificates will need to be valid until to be + useful to the requesting server. If not supplied, the current time as determined by the notary server is used. From 8f1a4ae0ea8cfb5192911f862c55c029aac175de Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 16:43:51 -0600 Subject: [PATCH 240/308] Formatting --- specification/server_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 4cf92e9b..3bdc5b2b 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -111,7 +111,7 @@ Retrieving Server Keys specification due to lack of significance. It may be reviewed `here `_. -Each homeserver publishes its public keys under ``/_matrix/key/v2/server/{keyId}`. +Each homeserver publishes its public keys under ``/_matrix/key/v2/server/{keyId}``. Homeservers query for keys by either getting ``/_matrix/key/v2/server/{keyId}`` directly or by querying an intermediate notary server using a ``/_matrix/key/v2/query/{serverName}/{keyId}`` API. Intermediate notary servers From e27f4a69a05a893ca5e5aec6df4269d6f93fcdac Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 16:46:13 -0600 Subject: [PATCH 241/308] Key versions must be [0-9a-zA-Z_] --- api/server-server/definitions/keys.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/server-server/definitions/keys.yaml b/api/server-server/definitions/keys.yaml index 67908268..5fba4d18 100644 --- a/api/server-server/definitions/keys.yaml +++ b/api/server-server/definitions/keys.yaml @@ -29,7 +29,8 @@ properties: The object's key is the algorithm and version combined (``ed25519`` being the algorithm and ``abc123`` being the version in the example below). Together, - this forms the Key ID. + this forms the Key ID. The version must have characters matching the regular + expression ``[a-zA-Z0-9_]``. required: true additionalProperties: type: object @@ -52,7 +53,8 @@ properties: The object's key is the algorithm and version combined (``ed25519`` being the algorithm and ``0ldK3y`` being the version in the example below). Together, - this forms the Key ID. + this forms the Key ID. The version must have characters matching the regular + expression ``[a-zA-Z0-9_]``. additionalProperties: type: object title: Old Verify Key From cafd1a9ab3ebcac646a411f3fb0bc11725d61d04 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 26 Jul 2018 16:46:33 -0600 Subject: [PATCH 242/308] Use more modern timestamps --- api/server-server/definitions/keys.yaml | 4 ++-- api/server-server/examples/server_key.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/server-server/definitions/keys.yaml b/api/server-server/definitions/keys.yaml index 5fba4d18..738e9e46 100644 --- a/api/server-server/definitions/keys.yaml +++ b/api/server-server/definitions/keys.yaml @@ -60,7 +60,7 @@ properties: title: Old Verify Key example: { "ed25519:0ldK3y": { - "expired_ts": 922834800000, + "expired_ts": 1532645052628, "key": "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg" } } @@ -70,7 +70,7 @@ properties: format: int64 description: POSIX timestamp in milliseconds for when this key expired. required: true - example: 922834800000 + example: 1532645052628 key: type: string description: The `Unpadded Base64`_ encoded key. diff --git a/api/server-server/examples/server_key.json b/api/server-server/examples/server_key.json index 8a82d76e..a7e4621b 100644 --- a/api/server-server/examples/server_key.json +++ b/api/server-server/examples/server_key.json @@ -7,7 +7,7 @@ }, "old_verify_keys": { "ed25519:0ldk3y": { - "expired_ts": 922834800000, + "expired_ts": 1532645052628, "key": "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg==" } }, @@ -19,5 +19,5 @@ "tls_fingerprints": [{ "sha256": "VGhpcyBpcyBoYXNoIHdoaWNoIHNob3VsZCBiZSBieXRlcw==" }], - "valid_until_ts": 1052262000000 + "valid_until_ts": 1652262000000 } \ No newline at end of file From e4fd088fcc7a9357c4551c544a0e84b4d0f3731e Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 27 Jul 2018 12:05:30 +0100 Subject: [PATCH 243/308] Fix room tags spec --- api/client-server/tags.yaml | 6 +++-- specification/modules/tags.rst | 48 ++++++++++++++++------------------ 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/api/client-server/tags.yaml b/api/client-server/tags.yaml index 9310b25f..42db5d9d 100644 --- a/api/client-server/tags.yaml +++ b/api/client-server/tags.yaml @@ -1,4 +1,5 @@ # Copyright 2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -64,8 +65,9 @@ paths: examples: application/json: { "tags": { - "work": {"order": "1"}, - "pinned": {} + "m.favourite": {} + "u.Work": {"order": "1"}, + "u.Customers": {} } } tags: diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 25e48ab3..c085391e 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -1,4 +1,5 @@ .. Copyright 2016 OpenMarket Ltd +.. Copyright 2018 New Vector Ltd .. .. Licensed under the Apache License, Version 2.0 (the "License"); .. you may not use this file except in compliance with the License. @@ -17,22 +18,19 @@ Room Tagging .. _module:tagging: -Users can add tags to rooms. Tags are short strings used to label rooms, e.g. -"work", "family". A room may have multiple tags. Tags are only visible to the -user that set them but are shared across all their devices. +Users can add tags to rooms. Tags are namespaced strings used to label rooms. +A room may have multiple tags. Tags are only visible to the user that set them +but are shared across all their devices. Events ------ The tags on a room are received as single ``m.tag`` event in the -``account_data`` section of a room in a ``/sync``. +``account_data`` section of a room. The content of the ``m.tag`` event is a +``tags`` key whose value is an object mapping the name of each tag to another +object. -The ``m.tag`` can also be received in a ``/events`` response or in the -``account_data`` section of a room in ``/initialSync``. ``m.tag`` -events appearing in ``/events`` will have a ``room_id`` with the room -the tags are for. - -Each tag has an associated JSON object with information about the tag, e.g how +The JSON object associated with each tag gives information about the tag, e.g how to order the rooms with a given tag. Ordering information is given under the ``order`` key as a number between 0 and @@ -43,25 +41,25 @@ after the rooms with that tag that have an ``order`` key. The name of a tag MUST not exceed 255 bytes. -The name of a tag should be human readable. When displaying tags for a room a -client should display this human readable name. When adding a tag for a room -a client may offer a list to choose from that includes all the tags that the -user has previously set on any of their rooms. - -Two special names are listed in the specification: - -* ``m.favourite`` -* ``m.lowpriority`` - -{{m_tag_event}} - Tags namespaces are defined in the following way, depending on how the client are expected to interpret them: -* The namespace ``m.*`` is reserved for tags defined in the current specification -* The namespace ``u.*`` is reserved for user-defined tags, and the client should not try to interpret as anything other than an utf8 string +* The namespace ``m.*`` is reserved for tags defined in the Matrix specification +* The namespace ``u.*`` is reserved for user-defined tags. The portion of the string after the ``u.`` + is defined to be the display name of this tag. No other semantics should be inferred from tags in + this namespace. * A client or app willing to use special tags for advanced functionnality should namespace them similarly to state keys: ``tld.name.*`` * Any tag in the ``tld.name.*`` form but not matching the namespace of the current client should be ignored -* Any tag not matching the previous rules should be interpreted as an user tag from the ``u.*`` namespace +* Any tag not matching the above rules should be interpreted as a user tag from the ``u.*`` namespace, as if + the name had already had ``u.`` stripped from the start (ie. the name of the tag is used as the + display name directly). + +Two special names are listed in the specification: +The following tags are defined in the ``m.*`` namespace: + +* ``m.favourite``: The user's favourite rooms. These should be shown with higher precedence than other rooms. +* ``m.lowpriority``: These should be shown with lower precedence than others. + +{{m_tag_event}} Client Behaviour ---------------- From f90ed4b77d19363667e11f8dd4555a49e3fe3700 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 27 Jul 2018 12:08:04 +0100 Subject: [PATCH 244/308] Make non-namespaced tags verboten --- specification/modules/tags.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index c085391e..bf985ede 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -51,7 +51,8 @@ Tags namespaces are defined in the following way, depending on how the client ar * Any tag in the ``tld.name.*`` form but not matching the namespace of the current client should be ignored * Any tag not matching the above rules should be interpreted as a user tag from the ``u.*`` namespace, as if the name had already had ``u.`` stripped from the start (ie. the name of the tag is used as the - display name directly). + display name directly). These non-namespaced tags are supported for historical reasons. New tags should use + one of the defined namespaces above. Two special names are listed in the specification: The following tags are defined in the ``m.*`` namespace: From 5ddf5374db3dc80fcb093946ff5c6c5465b5e75f Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 27 Jul 2018 12:24:29 +0100 Subject: [PATCH 245/308] Missing comma --- api/client-server/tags.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client-server/tags.yaml b/api/client-server/tags.yaml index 42db5d9d..b7bafab6 100644 --- a/api/client-server/tags.yaml +++ b/api/client-server/tags.yaml @@ -65,7 +65,7 @@ paths: examples: application/json: { "tags": { - "m.favourite": {} + "m.favourite": {}, "u.Work": {"order": "1"}, "u.Customers": {} } From 2dc51d416dddad4932a78c9b0e1128fe68b985de Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 27 Jul 2018 12:27:30 +0100 Subject: [PATCH 246/308] Ignore stuff in `m.` you don't understand --- specification/modules/tags.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index bf985ede..8c74c55f 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -43,7 +43,8 @@ The name of a tag MUST not exceed 255 bytes. Tags namespaces are defined in the following way, depending on how the client are expected to interpret them: -* The namespace ``m.*`` is reserved for tags defined in the Matrix specification +* The namespace ``m.*`` is reserved for tags defined in the Matrix specification. Clients must ignore + any tags in this namespace they don't understand. * The namespace ``u.*`` is reserved for user-defined tags. The portion of the string after the ``u.`` is defined to be the display name of this tag. No other semantics should be inferred from tags in this namespace. From 73b0a03b9bb72d965afa235f8766101398a2057a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 27 Jul 2018 08:39:15 -0600 Subject: [PATCH 247/308] Remove more padding --- api/server-server/examples/server_key.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/server-server/examples/server_key.json b/api/server-server/examples/server_key.json index a7e4621b..bebd8445 100644 --- a/api/server-server/examples/server_key.json +++ b/api/server-server/examples/server_key.json @@ -2,22 +2,22 @@ "server_name": "example.org", "verify_keys": { "ed25519:abc123": { - "key": "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA==" + "key": "VGhpcyBzaG91bGQgYmUgYSByZWFsIGVkMjU1MTkgcGF5bG9hZA" } }, "old_verify_keys": { "ed25519:0ldk3y": { "expired_ts": 1532645052628, - "key": "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg==" + "key": "VGhpcyBzaG91bGQgYmUgeW91ciBvbGQga2V5J3MgZWQyNTUxOSBwYXlsb2FkLg" } }, "signatures": { "example.org": { - "ed25519:auto2": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU=" + "ed25519:auto2": "VGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgYSBzaWduYXR1cmU" } }, "tls_fingerprints": [{ - "sha256": "VGhpcyBpcyBoYXNoIHdoaWNoIHNob3VsZCBiZSBieXRlcw==" + "sha256": "VGhpcyBpcyBoYXNoIHdoaWNoIHNob3VsZCBiZSBieXRlcw" }], "valid_until_ts": 1652262000000 } \ No newline at end of file From 31687608dc90c6a77f4e39531576d59edd505d00 Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Sun, 29 Jul 2018 12:16:10 +0900 Subject: [PATCH 248/308] Factor out common code --- api/check_examples.py | 57 ++++++++++++++++--------------------------- 1 file changed, 21 insertions(+), 36 deletions(-) diff --git a/api/check_examples.py b/api/check_examples.py index fb9430b6..81a9298e 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -43,22 +43,25 @@ except ImportError as e: raise +def check_schema(filepath, example, schema): + # Setting the 'id' tells jsonschema where the file is so that it + # can correctly resolve relative $ref references in the schema + schema['id'] = "file://" + os.path.abspath(filepath) + example = resolve_references(filepath, example) + resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file}) + jsonschema.validate(example, schema, resolver=resolver) + + def check_parameter(filepath, request, parameter): schema = parameter.get("schema") example = schema.get('example') - fileurl = "file://" + os.path.abspath(filepath) if example and schema: try: print ("Checking request schema for: %r %r" % ( filepath, request )) - # Setting the 'id' tells jsonschema where the file is so that it - # can correctly resolve relative $ref references in the schema - schema['id'] = fileurl - example = resolve_references(filepath, example) - resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file}) - jsonschema.validate(example, schema, resolver=resolver) + check_schema(filepath, example, schema) except Exception as e: raise ValueError("Error validating JSON schema for %r" % ( request @@ -68,18 +71,12 @@ def check_parameter(filepath, request, parameter): def check_response(filepath, request, code, response): example = response.get('examples', {}).get('application/json') schema = response.get('schema') - fileurl = "file://" + os.path.abspath(filepath) if example and schema: try: print ("Checking response schema for: %r %r %r" % ( filepath, request, code )) - # Setting the 'id' tells jsonschema where the file is so that it - # can correctly resolve relative $ref references in the schema - schema['id'] = fileurl - example = resolve_references(filepath, example) - resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file}) - jsonschema.validate(example, schema, resolver=resolver) + check_schema(filepath, example, schema) except Exception as e: raise ValueError("Error validating JSON schema for %r %r" % ( request, code @@ -127,30 +124,18 @@ def resolve_references(path, schema): return schema -def load_yaml(path): - if not path.startswith("file:///"): - raise Exception("Bad ref: %s" % (path,)) - path = path[len("file://"):] - with open(path, "r") as f: - return yaml.load(f) - - -def load_json(path): - if not path.startswith("file:///"): - raise Exception("Bad ref: %s" % (path,)) - path = path[len("file://"):] - with open(path, "r") as f: - return json.load(f) - - def load_file(path): print("Loading reference: %s" % path) - if path.endswith(".json"): - return load_json(path) - else: - # We have to assume it's YAML because some of the YAML examples - # do not have file extensions. - return load_yaml(path) + if not path.startswith("file:///"): + raise Exception("Bad ref: %s" % (path,)) + path = path[len("file://"):] + with open(path, "r") as f: + if path.endswith(".json"): + return json.load(f) + else: + # We have to assume it's YAML because some of the YAML examples + # do not have file extensions. + return yaml.load(f) if __name__ == '__main__': From 2c9f00d37aef000fc0a483d9d4bccac973124adb Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Sun, 29 Jul 2018 12:23:41 +0900 Subject: [PATCH 249/308] Follow PEP 8 --- api/check_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/check_examples.py b/api/check_examples.py index 81a9298e..3e791bae 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -58,7 +58,7 @@ def check_parameter(filepath, request, parameter): if example and schema: try: - print ("Checking request schema for: %r %r" % ( + print("Checking request schema for: %r %r" % ( filepath, request )) check_schema(filepath, example, schema) From d17ec7f1846a96844c750dac3935ccd32b323b74 Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Sun, 29 Jul 2018 12:25:11 +0900 Subject: [PATCH 250/308] Check that path starts with file://, not file:/// Because file:/// is not a thing on Windows. --- api/check_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/check_examples.py b/api/check_examples.py index 3e791bae..eb861fca 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -126,7 +126,7 @@ def resolve_references(path, schema): def load_file(path): print("Loading reference: %s" % path) - if not path.startswith("file:///"): + if not path.startswith("file://"): raise Exception("Bad ref: %s" % (path,)) path = path[len("file://"):] with open(path, "r") as f: From c6bfac41c535a9608a67181f43864bafd3ae6d16 Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Sun, 29 Jul 2018 12:30:25 +0900 Subject: [PATCH 251/308] Bypass jsonschema references resolver jsonschema's resolver doesn't work with local files on Windows. resolve_references only works with the local files (on Windows as well) but that's exactly what's needed for the case (as long as we don't have remote references, that is). Signed-off-by: Alexey Rusakov --- api/check_examples.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/api/check_examples.py b/api/check_examples.py index eb861fca..cb85cb50 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -44,16 +44,14 @@ except ImportError as e: def check_schema(filepath, example, schema): - # Setting the 'id' tells jsonschema where the file is so that it - # can correctly resolve relative $ref references in the schema - schema['id'] = "file://" + os.path.abspath(filepath) example = resolve_references(filepath, example) + schema = resolve_references(filepath, schema) resolver = jsonschema.RefResolver(filepath, schema, handlers={"file": load_file}) jsonschema.validate(example, schema, resolver=resolver) def check_parameter(filepath, request, parameter): - schema = parameter.get("schema") + schema = parameter.get("schema", {}) example = schema.get('example') if example and schema: From 346db48588612620f179e0b9560ac4d1d8a24729 Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Sun, 29 Jul 2018 14:04:07 +0900 Subject: [PATCH 252/308] Revert giving get() a default As per the PR review. Signed-off-by: Alexey Rusakov --- api/check_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/check_examples.py b/api/check_examples.py index cb85cb50..0fb275b1 100755 --- a/api/check_examples.py +++ b/api/check_examples.py @@ -51,7 +51,7 @@ def check_schema(filepath, example, schema): def check_parameter(filepath, request, parameter): - schema = parameter.get("schema", {}) + schema = parameter.get("schema") example = schema.get('example') if example and schema: From 0ff882fb3b3467daf74aca2fb55c4a4f12614272 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 13 Jul 2018 11:41:51 +0100 Subject: [PATCH 253/308] Field definitions for protocol metadata --- api/application-service/definitions/protocol_metadata.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/application-service/definitions/protocol_metadata.yaml b/api/application-service/definitions/protocol_metadata.yaml index 72264060..2b2c8f4e 100644 --- a/api/application-service/definitions/protocol_metadata.yaml +++ b/api/application-service/definitions/protocol_metadata.yaml @@ -13,6 +13,8 @@ # limitations under the License. type: object description: Dictionary of supported third party protocols. +additionalProperties: + $ref: protocol.yaml example: { "irc": { "user_fields": ["network", "nickname"], From 273acb277caa7020dbcea76e2707c45eca8670cd Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 13 Jul 2018 12:07:15 +0100 Subject: [PATCH 254/308] Change fields to use new extended query array syntax --- api/application-service/application_service.yaml | 4 ++-- api/client-server/third_party_lookup.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index b459e29f..138c75bd 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -270,7 +270,7 @@ paths: required: true x-example: irc - in: query - name: field1, field2... + name: fields... type: string description: |- One or more custom fields that are passed to the application @@ -321,7 +321,7 @@ paths: required: true x-example: irc - in: query - name: field1, field2... + name: field... type: string description: |- One or more custom fields that are passed to the application diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 55d4c70a..226eec21 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -122,7 +122,7 @@ paths: required: true x-example: irc - in: query - name: field1, field2... + name: fields... type: string description: |- One or more custom fields that are passed to the AS to help identify the user. From 3e4962f21152e31bce6a3ff44bb0c65b11c89616 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 13 Jul 2018 12:25:48 +0100 Subject: [PATCH 255/308] Document standardized extensions to OpenAPI v2 --- api/openapi_extensions.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 api/openapi_extensions.rst diff --git a/api/openapi_extensions.rst b/api/openapi_extensions.rst new file mode 100644 index 00000000..75c48687 --- /dev/null +++ b/api/openapi_extensions.rst @@ -0,0 +1,23 @@ +OpenAPI Extensions +================== + +For some functionality that is not directly provided by the OpenAPI v2 +specification, some extensions have been added that are to be consistent +across the specification. The defined extensions are listed below. Extensions +should not break parsers, however if extra functionality is required, aware +parsers should be able to take advantage of the added syntax. + +Extensible Query Parameters +--------------------------- + +If a unknown amount of query parameters can be added to a request, the +``name`` should be in form of ``something...``, with the trailing ellipses +representing the possibility of more fields. + +Example: + +.. code-block:: + + - in: query + name: fields... + type: string \ No newline at end of file From 53137a3c5318c1a8e50a988e7c511302cf923590 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 23 Jul 2018 14:38:52 +0100 Subject: [PATCH 256/308] c/p bug, fix operationIds, move rst docs to md --- .../application_service.yaml | 2 +- api/application-service/definitions/user.yaml | 2 +- api/client-server/third_party_lookup.yaml | 9 +++++---- ...pi_extensions.rst => openapi_extensions.md} | 18 ++++++++---------- 4 files changed, 15 insertions(+), 16 deletions(-) rename api/{openapi_extensions.rst => openapi_extensions.md} (65%) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index 138c75bd..6f2eed16 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -214,7 +214,7 @@ paths: This API is called by the homeserver when it wants to present clients with specific information about the various third party networks that an application service supports. - operationId: queryMetadata + operationId: getProtocolMetadata parameters: - in: path name: protocol diff --git a/api/application-service/definitions/user.yaml b/api/application-service/definitions/user.yaml index 5f8d0460..a7b2287e 100644 --- a/api/application-service/definitions/user.yaml +++ b/api/application-service/definitions/user.yaml @@ -27,5 +27,5 @@ properties: type: object example: "user": "jim" -title: Location +title: User type: object \ No newline at end of file diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 226eec21..ee14a137 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -34,7 +34,7 @@ paths: Fetches the overall metadata about protocols supported by the homeserver. Includes both the available protocols and all fields required for queries against each protocol. - operationId: queryMetadata + operationId: getProtocols responses: 200: description: The protocols supported by the homeserver. @@ -45,7 +45,7 @@ paths: summary: Retrieve metadata about a specific protocol that the homeserver supports. description: |- Fetches the metadata from the homeserver about a particular third party protocol. - operationId: queryMetadata + operationId: getProtocolMetadata parameters: - in: path name: protocol @@ -122,8 +122,9 @@ paths: required: true x-example: irc - in: query - name: fields... - type: string + name: fields + explode: true + type: object description: |- One or more custom fields that are passed to the AS to help identify the user. responses: diff --git a/api/openapi_extensions.rst b/api/openapi_extensions.md similarity index 65% rename from api/openapi_extensions.rst rename to api/openapi_extensions.md index 75c48687..4a01c477 100644 --- a/api/openapi_extensions.rst +++ b/api/openapi_extensions.md @@ -1,5 +1,4 @@ -OpenAPI Extensions -================== +# OpenAPI Extensions For some functionality that is not directly provided by the OpenAPI v2 specification, some extensions have been added that are to be consistent @@ -7,17 +6,16 @@ across the specification. The defined extensions are listed below. Extensions should not break parsers, however if extra functionality is required, aware parsers should be able to take advantage of the added syntax. -Extensible Query Parameters ---------------------------- +## Extensible Query Parameters -If a unknown amount of query parameters can be added to a request, the -``name`` should be in form of ``something...``, with the trailing ellipses -representing the possibility of more fields. +If a unknown amount of query parameters can be added to a request, the `name` +must be `fields...`, with the trailing ellipses representing the possibility +of more fields. Example: -.. code-block:: - +``` - in: query name: fields... - type: string \ No newline at end of file + type: string +``` \ No newline at end of file From d785ac78a355c85f68c82e3ff5e558378d2d1bc0 Mon Sep 17 00:00:00 2001 From: user Date: Mon, 30 Jul 2018 11:01:27 +0100 Subject: [PATCH 257/308] Remove explode and replace with TODO, require auth on CS --- api/client-server/third_party_lookup.yaml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index ee14a137..293a105b 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -35,6 +35,8 @@ paths: homeserver. Includes both the available protocols and all fields required for queries against each protocol. operationId: getProtocols + security: + - accessToken: [] responses: 200: description: The protocols supported by the homeserver. @@ -46,6 +48,8 @@ paths: description: |- Fetches the metadata from the homeserver about a particular third party protocol. operationId: getProtocolMetadata + security: + - accessToken: [] parameters: - in: path name: protocol @@ -80,6 +84,8 @@ paths: identifier. It should attempt to canonicalise the identifier as much as reasonably possible given the network type. operationId: queryLocationByProtocol + security: + - accessToken: [] parameters: - in: path name: protocol @@ -113,6 +119,8 @@ paths: Retrieve a Matrix User ID linked to a user on the third party service, given a set of user parameters. operationId: queryUserByProtocol + security: + - accessToken: [] parameters: - in: path name: protocol @@ -121,10 +129,10 @@ paths: The name of the protocol. required: true x-example: irc + # TODO: Change to 'explode' after OpenAPI/Swagger v3 update - in: query - name: fields - explode: true - type: object + name: field1, field2... + type: string description: |- One or more custom fields that are passed to the AS to help identify the user. responses: @@ -147,6 +155,8 @@ paths: Retreive an array of third party network locations from a Matrix room alias. operationId: queryLocationByAlias + security: + - accessToken: [] parameters: - in: query name: alias @@ -173,6 +183,8 @@ paths: description: |- Retreive an array of third party users from a Matrix User ID. operationId: queryUserByID + security: + - accessToken: [] parameters: - in: query name: userid @@ -192,4 +204,4 @@ paths: "errcode": "M_NOT_FOUND" } schema: - $ref: definitions/errors/error.yaml \ No newline at end of file + $ref: definitions/errors/error.yaml From a2e2ced03b1754ca71d69def19eb03e2deeb9766 Mon Sep 17 00:00:00 2001 From: user Date: Mon, 30 Jul 2018 11:09:13 +0100 Subject: [PATCH 258/308] Add reverse-lookup examples. --- api/client-server/third_party_lookup.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 293a105b..55b34a81 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -163,6 +163,7 @@ paths: type: string description: The Matrix room alias to look up. required: true + x-example: "#matrix:matrix.org" responses: 200: description: |- @@ -191,6 +192,7 @@ paths: type: string description: The Matrix User ID to look up. required: true + x-example: "@bob:matrix.org" responses: 200: description: |- From 5f8967c0742faa108f994771e2ecc0b0b42854da Mon Sep 17 00:00:00 2001 From: user Date: Mon, 30 Jul 2018 19:19:35 +0100 Subject: [PATCH 259/308] Move TODO. field1, field2... -> fields... --- api/application-service/application_service.yaml | 4 ++-- api/client-server/third_party_lookup.yaml | 3 +-- api/openapi_extensions.md | 4 +++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api/application-service/application_service.yaml b/api/application-service/application_service.yaml index 6f2eed16..42c0c0cf 100644 --- a/api/application-service/application_service.yaml +++ b/api/application-service/application_service.yaml @@ -321,7 +321,7 @@ paths: required: true x-example: irc - in: query - name: field... + name: fields... type: string description: |- One or more custom fields that are passed to the application @@ -446,4 +446,4 @@ paths: "errcode": "COM.EXAMPLE.MYAPPSERVICE_NOT_FOUND" } schema: - $ref: ../client-server/definitions/errors/error.yaml \ No newline at end of file + $ref: ../client-server/definitions/errors/error.yaml diff --git a/api/client-server/third_party_lookup.yaml b/api/client-server/third_party_lookup.yaml index 55b34a81..cba9ce22 100644 --- a/api/client-server/third_party_lookup.yaml +++ b/api/client-server/third_party_lookup.yaml @@ -129,9 +129,8 @@ paths: The name of the protocol. required: true x-example: irc - # TODO: Change to 'explode' after OpenAPI/Swagger v3 update - in: query - name: field1, field2... + name: fields... type: string description: |- One or more custom fields that are passed to the AS to help identify the user. diff --git a/api/openapi_extensions.md b/api/openapi_extensions.md index 4a01c477..9f4745fd 100644 --- a/api/openapi_extensions.md +++ b/api/openapi_extensions.md @@ -8,6 +8,8 @@ parsers should be able to take advantage of the added syntax. ## Extensible Query Parameters + + If a unknown amount of query parameters can be added to a request, the `name` must be `fields...`, with the trailing ellipses representing the possibility of more fields. @@ -18,4 +20,4 @@ Example: - in: query name: fields... type: string -``` \ No newline at end of file +``` From 7679b4f1d168e4f5ab7c2e94d34923079782f234 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 30 Jul 2018 16:58:13 -0600 Subject: [PATCH 260/308] Improve documentation on how Transactions work The response is based upon various sections of the Synapse code in how it generates a response. There are no new fields added to the transaction. Originally, `previous_ids` and `pdu_failures` were to be documented however neither of these are used in the real world. --- .../definitions/transaction.yaml | 8 ++- api/server-server/transactions.yaml | 58 +++++++++++++++++-- specification/server_server_api.rst | 38 +----------- 3 files changed, 58 insertions(+), 46 deletions(-) diff --git a/api/server-server/definitions/transaction.yaml b/api/server-server/definitions/transaction.yaml index 930ddec1..7df8b646 100644 --- a/api/server-server/definitions/transaction.yaml +++ b/api/server-server/definitions/transaction.yaml @@ -25,11 +25,13 @@ properties: origin_server_ts: type: integer format: int64 - description: Timestamp in milliseconds on originating homeserver when this transaction started. - example: 1234567890 + description: |- + POSIX timestamp in milliseconds on originating homeserver when this + transaction started. + example: 1532991320875 pdus: type: array description: List of persistent updates to rooms. items: $ref: "pdu.yaml" -required: ['origin', 'origin_server_ts', 'pdus'] \ No newline at end of file +required: ['origin', 'origin_server_ts', 'pdus'] diff --git a/api/server-server/transactions.yaml b/api/server-server/transactions.yaml index 13ba6826..2a9180a1 100644 --- a/api/server-server/transactions.yaml +++ b/api/server-server/transactions.yaml @@ -30,16 +30,18 @@ paths: Push messages representing live activity to another server. The destination name will be set to that of the receiving server itself. Each embedded PDU in the transaction body will be processed. + + The sending server must wait and retry for a 200 OK response before sending a + transaction with a different ``txnId`` to the receiving server. operationId: sendTransaction parameters: - in: path name: txnId type: string - # TODO: "Overrides any ID given in the JSON body" - What does this mean? description: |- - The transaction ID. Overrides any ID given in the JSON body. + The transaction ID. required: true - x-example: TODO # No examples in the spec so far + x-example: S0meTransacti0nId - in: body name: body type: object @@ -51,7 +53,9 @@ paths: properties: edus: type: array - description: List of ephemeral messages. May be omitted if there are no ephemeral messages to be sent. + description: |- + List of ephemeral messages. May be omitted if there are no ephemeral + messages to be sent. items: $ref: "definitions/edu.yaml" example: { @@ -60,5 +64,47 @@ paths: } responses: 200: - # TODO: Spec this (and figure out what it is) - description: TODO + description: |- + The result of processing the transaction. The server is to use this response even in + the event of one or more PDUs failing to be processed. + schema: + type: array + minItems: 2 + maxItems: 2 + items: + - type: integer + description: The value ``200``. + example: 200 + - type: object + title: PDU Processing Results + description: The results for the processing of each PDU in the transaction. + properties: + pdus: + type: object + description: |- + The PDUs from the original transaction. The string key represents the ID of the + PDU (event) that was processed. + additionalProperties: + type: object + title: PDU Processing Result + description: Information about how the PDU was handled. + properties: + error: + type: string + description: |- + A human readable description about what went wrong in processing this PDU. + If no error is present, the PDU can be considered successfully handled. + example: "You are not allowed to send a message to this room." + required: ['pdus'] + examples: + application/json: [ + 200, + { + "pdus": { + "$successful_event:domain.com": {}, + "$failed_event:example.org": { + "error": "You are not allowed to send a message to this room." + } + } + } + ] diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f1825f27..a190d9d2 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -242,41 +242,7 @@ of Transaction messages, which are encoded as JSON objects, passed over an HTTP PUT request. A Transaction is meaningful only to the pair of homeservers that exchanged it; they are not globally-meaningful. -Each transaction has: - - An opaque transaction ID, unique among transactions from the same origin. - - A timestamp (UNIX epoch time in milliseconds) generated by its origin - server. - - An origin and destination server name. - - A list of PDUs and EDUs - the actual message payload that the Transaction - carries. - -Transaction Fields -~~~~~~~~~~~~~~~~~~ - -==================== =================== ====================================== - Key Type Description -==================== =================== ====================================== -``origin`` String **Required**. ``server_name`` of homeserver sending - this transaction. -``origin_server_ts`` Integer **Required**. Timestamp in milliseconds on - originating homeserver when this - transaction started. -``pdus`` List of Objects **Required**. List of persistent updates to rooms. -``edus`` List of Objects List of ephemeral messages. May be omitted - if there are no ephemeral messages to - be sent. -==================== =================== ====================================== - -Example: - -.. code:: json - - { - "origin_server_ts": 1404835423000, - "origin": "matrix.org", - "pdus": [...], - "edus": [...] - } +{{transactions_ss_http_api}} PDUs ---- @@ -682,8 +648,6 @@ All these URLs are name-spaced within a prefix of:: /_matrix/federation/v1/... -{{transactions_ss_http_api}} - {{events_ss_http_api}} {{query_general_ss_http_api}} From d7919b6989847297c0c29b00529d2b67bd9d64b0 Mon Sep 17 00:00:00 2001 From: Ben Parsons Date: Tue, 31 Jul 2018 15:08:42 +0100 Subject: [PATCH 261/308] also scrape PRs when the full URL is pasted --- scripts/proposals.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/proposals.py b/scripts/proposals.py index a8f58cae..d8c1aa24 100755 --- a/scripts/proposals.py +++ b/scripts/proposals.py @@ -138,10 +138,15 @@ for label in labels: pr_list_formatted = set() pr_list = pr_list.group(1) for p in pr_list.split(","): - if not re.match(r"#\d", p.strip()): + if re.match(r"#\d", p.strip()): + prs.add(p.strip()) + pr_list_formatted.add("`PR" + str(p.strip()) + "`_") + elif re.match(r"https://github.com/matrix-org/matrix-doc/pulls/\d", p.strip()): + pr = "#" + p.strip().replace('https://github.com/matrix-org/matrix-doc/pulls/', '') + prs.add(pr) + pr_list_formatted.add("`PR" + str(pr) + "`_") + else: raise RuntimeWarning - prs.add(p.strip()) - pr_list_formatted.add("`PR" + str(p.strip()) + "`_") text_file.write(" - " + ', '.join(pr_list_formatted)) text_file.write("\n") else: From ea307b5bdbae811e8b94e0603c2c0ac5fde9ba0f Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 31 Jul 2018 13:55:53 -0600 Subject: [PATCH 262/308] Support rendering schema definitions in the spec --- .../templating/matrix_templates/sections.py | 14 +++++++ .../templates/schema-definition.tmpl | 21 ++++++++++ scripts/templating/matrix_templates/units.py | 38 +++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 scripts/templating/matrix_templates/templates/schema-definition.tmpl diff --git a/scripts/templating/matrix_templates/sections.py b/scripts/templating/matrix_templates/sections.py index 64e32aa4..1a93c723 100644 --- a/scripts/templating/matrix_templates/sections.py +++ b/scripts/templating/matrix_templates/sections.py @@ -189,3 +189,17 @@ class MatrixSections(Sections): template = self.env.get_template("apis.tmpl") apis = self.units.get("apis") return template.render(apis=apis) + + def render_swagger_definition(self): + rendered = {} + template = self.env.get_template("schema-definition.tmpl") + subtitle_title_char = self.units.get("spec_targets")[ + "relative_title_styles" + ]["subtitle"] + definitions = self.units.get("swagger_definitions") + for group, swagger_def in definitions.items(): + rendered["definition_" + group] = template.render( + definition=swagger_def['definition'], + examples=swagger_def['examples'], + title_kind=subtitle_title_char) + return rendered \ No newline at end of file diff --git a/scripts/templating/matrix_templates/templates/schema-definition.tmpl b/scripts/templating/matrix_templates/templates/schema-definition.tmpl new file mode 100644 index 00000000..42a7ae47 --- /dev/null +++ b/scripts/templating/matrix_templates/templates/schema-definition.tmpl @@ -0,0 +1,21 @@ +{% import 'tables.tmpl' as tables -%} + +``{{definition.title}}`` Schema +{{(11 + definition.title | length) * title_kind}} + +{% if 'description' in definition %} +{{definition.description}} +{% endif %} + +{% for table in definition.tables -%} +{{"``"+table.title+"``" if table.title else "" }} +{{ tables.paramtable(table.rows) }} +{% endfor %} + +Example{% if examples | length > 1 %}s{% endif %}: + +{% for example in examples %} +.. code:: json + + {{example | jsonify(4, 4)}} +{% endfor %} diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 5b39a2b3..d6579e6c 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -43,6 +43,13 @@ HTTP_APIS = { os.path.join(matrix_doc_dir, "api/push-gateway"): "push", os.path.join(matrix_doc_dir, "api/server-server"): "ss", } +SWAGGER_DEFINITIONS = { + os.path.join(matrix_doc_dir, "api/application-service/definitions"): "as", + os.path.join(matrix_doc_dir, "api/client-server/definitions"): "cs", + #os.path.join(matrix_doc_dir, "api/identity/definitions"): "is", + #os.path.join(matrix_doc_dir, "api/push-gateway/definitions"): "push", + os.path.join(matrix_doc_dir, "api/server-server/definitions"): "ss", +} EVENT_EXAMPLES = os.path.join(matrix_doc_dir, "event-schemas/examples") EVENT_SCHEMA = os.path.join(matrix_doc_dir, "event-schemas/schema") CORE_EVENT_SCHEMA = os.path.join(matrix_doc_dir, "event-schemas/schema/core-event-schema") @@ -654,6 +661,37 @@ class MatrixUnits(Units): apis[group_name] = api return apis + + def load_swagger_definitions(self): + defs = {} + for path, prefix in SWAGGER_DEFINITIONS.items(): + for filename in os.listdir(path): + if not filename.endswith(".yaml"): + continue + filepath = os.path.join(path, filename) + logger.info("Reading swagger definition: %s" % filepath) + with open(filepath, "r") as f: + # strip .yaml + group_name = filename[:-5].replace("-", "_") + group_name = "%s_%s" % (prefix, group_name) + definition = yaml.load(f.read(), OrderedLoader) + definition = resolve_references(filepath, definition) + if 'type' not in definition: + continue + try: + example = get_example_for_schema(definition) + except: + pass # do nothing - we don't care + if 'title' not in definition: + definition['title'] = "NO_TITLE" + definition['tables'] = get_tables_for_schema(definition) + defs[group_name] = { + "definition": definition, + "examples": [example] if example is not None else [], + } + return defs + + def load_common_event_fields(self): """Parse the core event schema files From 5027a9a59a4ca257f6b71f78f6f0fe3811940400 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 31 Jul 2018 13:58:23 -0600 Subject: [PATCH 263/308] Improve documentation around EDUs and PDUs Clarify fields, improve examples, and make the tables in the spec be generated rather than duplicated. --- api/server-server/definitions/edu.yaml | 6 +- .../definitions/unsigned_pdu.yaml | 33 +++++- api/server-server/examples/edu.json | 4 +- api/server-server/examples/unsigned_pdu.json | 3 + specification/server_server_api.rst | 101 +----------------- 5 files changed, 44 insertions(+), 103 deletions(-) diff --git a/api/server-server/definitions/edu.yaml b/api/server-server/definitions/edu.yaml index c89573fe..bc1f0cce 100644 --- a/api/server-server/definitions/edu.yaml +++ b/api/server-server/definitions/edu.yaml @@ -16,7 +16,7 @@ type: object title: Ephemeral Data Unit -description: An ephemeral data unit +description: An ephemeral data unit. example: $ref: "../examples/edu.json" properties: @@ -24,17 +24,15 @@ properties: type: string description: The type of ephemeral message. example: "m.presence" - required: true origin: type: string description: The server name sending the ephemeral message. example: "matrix.org" - required: true destination: type: string description: The server name receiving the ephemeral message. example: "elsewhere.com" - required: true content: type: object description: The content of the ephemeral message. +required: ['edu_type', 'origin', 'destination', 'content'] \ No newline at end of file diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml index 0aeede51..dad01537 100644 --- a/api/server-server/definitions/unsigned_pdu.yaml +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -108,8 +108,39 @@ properties: example: "$def456:matrix.org" unsigned: type: object - description: Additional data added by the origin server but not covered by the ``signatures``. + title: Example Unsigned Data + description: |- + Additional data added by the origin server but not covered by the ``signatures``. More + keys than those defined here may be used. example: {"key": "value"} + properties: + age: + type: integer + description: The number of milliseconds that have passed since this message was sent. + example: 4612 + age_ts: + type: integer + format: int64 + description: The POSIX timestamp this message was sent at in milliseconds. + replaces_state: + type: string + description: The event ID of the state event this event replaces. + example: "$state_event:domain.com" + prev_sender: + type: string + description: The sender of the replaced state event. + example: "@someone:domain.com" + prev_content: + type: string + description: The content of the replaced state event. + example: { + "membership": "join", + "displayname": "Bob" + } + redacted_because: + type: string + description: A reason for why the event was redacted. + example: "Inappropriate content" required: - event_id - room_id diff --git a/api/server-server/examples/edu.json b/api/server-server/examples/edu.json index 95a7b55d..9ff40dc3 100644 --- a/api/server-server/examples/edu.json +++ b/api/server-server/examples/edu.json @@ -1,7 +1,7 @@ { "edu_type": "m.presence", - "origin": "blue", - "destination": "orange", + "origin": "matrix.org", + "destination": "elsewhere.com", "content": { "key": "value" } diff --git a/api/server-server/examples/unsigned_pdu.json b/api/server-server/examples/unsigned_pdu.json index 0b585a63..f4d2e749 100644 --- a/api/server-server/examples/unsigned_pdu.json +++ b/api/server-server/examples/unsigned_pdu.json @@ -20,5 +20,8 @@ ], "content": { "key": "value" + }, + "unsigned": { + "age": 4612 } } \ No newline at end of file diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f1825f27..3ac15fa7 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -284,78 +284,6 @@ PDUs Each PDU contains a single Room Event which the origin server wants to send to the destination. - -PDU Fields -~~~~~~~~~~ - -.. TODO-spec - - Figure out how to embed swagger definitions in here (or improve the section) - -==================== ================== ======================================= - Key Type Description -==================== ================== ======================================= -``room_id`` String **Required**. Room identifier. -``sender`` String **Required**. The ID of the user sending - the event. -``origin`` String **Required**. ``server_name`` of the - homeserver that created this event. -``event_id`` String **Required**. Unique identifier for the - event being sent. -``origin_server_ts`` Integer **Required**. Timestamp in milliseconds - on origin homeserver when this event - was created. -``type`` String **Required**. Event type. -``state_key`` String If this key is present, the event is a - state event, and it will replace - previous events with the same ``type`` - and ``state_key`` in the room state. -``content`` Object **Required**. The content of the event. -``prev_events`` List of (String, **Required**. Event IDs and hashes of - {String: String}) the most recent events in the room that - pairs the homeserver was aware of when it - made this event. -``depth`` Integer **Required**. The maximum depth of the - ``prev_events``, plus one. -``auth_events`` List of (String, **Required**. Event IDs and hashes for - {String: String}) the "auth events" of this event. - pairs -``hashes`` {String: String} **Required**. Hashes of the PDU, - following the algorithm specified in - `Signing Events`_. -``signatures`` {String: **Required**. Signatures of the redacted - {String: String}} PDU, following the algorithm specified - in `Signing Events`_. -``redacts`` String For redaction events, the ID of the - event being redacted. -``unsigned`` Object Additional data added by the origin - server but not covered by the - ``signatures``. -==================== ================== ======================================= - -Example: - -.. code:: json - - { - "room_id": "!UcYsUzyxTGDxLBEvLy:example.org", - "sender": "@alice:example.com", - "origin": "example.com", - "event_id": "$a4ecee13e2accdadf56c1025:example.com", - "origin_server_ts": 1404838188000, - "type": "m.room.message", - "prev_events": [ - ["$af232176:example.org", {"sha256": "abase64encodedsha256hashshouldbe43byteslong"}] - ], - "hashes": {"sha256": "thishashcoversallfieldsincasethisisredacted"}, - "signatures": { - "example.com": { - "ed25519:key_version:": "these86bytesofbase64signaturecoveressentialfieldsincludinghashessocancheckredactedpdus" - } - }, - "content": {...} - } - The ``prev_events`` field of a PDU identifies the "parents" of the event, and thus establishes a partial ordering on events within the room by linking them into a Directed Acyclic Graph (DAG). The sending server should populate this @@ -386,6 +314,8 @@ following subset of the room state: - The current ``m.room.join_rules`` event, if any. - The sender's current ``m.room.member`` event, if any. +{{definition_ss_pdu}} + Authorization of PDUs ~~~~~~~~~~~~~~~~~~~~~ @@ -555,32 +485,11 @@ The rules are as follows: EDUs ---- -.. WARNING:: - This section may be misleading or inaccurate. - EDUs, by comparison to PDUs, do not have an ID, a room ID, or a list of -"previous" IDs. The only mandatory fields for these are the type, origin and -destination homeserver names, and the actual nested content. +"previous" IDs. They are intended to be non-persistent data such as user +presence, typing notifications, etc. -======================== ============ ========================================= - Key Type Description -======================== ============ ========================================= -``edu_type`` String The type of the ephemeral message. -``origin`` String The server name sending the ephemeral - message. -``destination`` String The server name receiving the ephemeral - message. -``content`` Object Content of the ephemeral message. -======================== ============ ========================================= - -.. code:: json - - { - "edu_type": "m.presence", - "origin": "blue", - "destination": "orange", - "content": {...} - } +{{definition_ss_edu}} Room State Resolution --------------------- From d0e8df8f3f23df0f2c2c9d60fd9db7029668a7fd Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 31 Jul 2018 14:02:04 -0600 Subject: [PATCH 264/308] prev_content is an object --- api/server-server/definitions/unsigned_pdu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml index dad01537..ab6e5b1c 100644 --- a/api/server-server/definitions/unsigned_pdu.yaml +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -131,7 +131,7 @@ properties: description: The sender of the replaced state event. example: "@someone:domain.com" prev_content: - type: string + type: object description: The content of the replaced state event. example: { "membership": "join", From 5aef545128b08b12c44a73fa92b903caf162db48 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 1 Aug 2018 13:31:23 -0600 Subject: [PATCH 265/308] Improve documentation for backfilling rooms There's not a whole lot to improve here - most of the changes are about reorganization and minor clarifications. --- api/server-server/backfill.yaml | 77 +++++++++++++++++++++++++++++ api/server-server/events.yaml | 32 ------------ specification/server_server_api.rst | 27 +++------- 3 files changed, 83 insertions(+), 53 deletions(-) create mode 100644 api/server-server/backfill.yaml diff --git a/api/server-server/backfill.yaml b/api/server-server/backfill.yaml new file mode 100644 index 00000000..f83057ae --- /dev/null +++ b/api/server-server/backfill.yaml @@ -0,0 +1,77 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Events API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/backfill/{roomId}": + get: + summary: Retrieves the events which precede the given event + description: |- + Retreives a sliding-window history of previous PDUs that occurred in the given room. + Starting from the PDU ID(s) given in the ``v`` argument, the PDUs that preceded it + are retrieved, up to the total number given by the ``limit``. + operationId: backfillRoom + parameters: + - in: path + name: roomId + type: string + description: The room ID to backfill. + required: true + x-example: "!SomeRoom:matrix.org" + - in: query + name: v + type: array + items: + type: string + description: The event IDs to backfill from. + required: true + x-example: ["$abc123:matrix.org"] + - in: query + name: limit + type: integer + description: The maximum number of PDUs to retrieve, including the given events. + required: true + x-example: 2 + responses: + 200: + description: |- + A transaction containing the PDUs that preceded the given event(s), including the given + event(s), up to the given limit. + schema: + $ref: "definitions/transaction.yaml" + # Override the example to show the response of the request a bit better + examples: + application/json: { + "$ref": "examples/transaction.json", + "pdus": [ + { + "$ref": "pdu.json", + "room_id": "!SomeRoom:matrix.org", + "event_id": "$abc123:matrix.org" + }, + { + "$ref": "pdu.json", + "room_id": "!SomeRoom:matrix.org" + }, + ] + } diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml index d540085d..3b48f15c 100644 --- a/api/server-server/events.yaml +++ b/api/server-server/events.yaml @@ -59,35 +59,3 @@ paths: description: A transaction containing a single PDU which is the event requested. schema: $ref: "definitions/transaction.yaml" - "/backfill/{roomId}": - get: - summary: Retrieves the events which precede the given event - description: |- - Retreives a sliding-window history of previous PDUs that occurred in the given room. - Starting from the PDU ID(s) given in the ``v`` argument, the PDUs that preceded it - are retrieved, up to the total number given by the ``limit``. - operationId: backfillRoom - parameters: - - in: path - name: roomId - type: string - description: The room ID to backfill. - required: true - x-example: "!abc123:matrix.org" - - in: query - name: v - type: string # TODO: The description says this is plural - figure out how to specify multiple, and spec it - description: The event ID to backfill from. - required: true - x-example: "$abc123:matrix.org" - - in: query - name: limit - type: integer - description: The maximum number of events to retrieve. - required: true - x-example: 10 - responses: - 200: - description: A transaction containing the PDUs that preceded the given event(s). - schema: - $ref: "definitions/transaction.yaml" diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index f1825f27..090885bc 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -850,32 +850,17 @@ To cover this case, the federation API provides a server-to-server analog of the ``/messages`` client API, allowing one homeserver to fetch history from another. This is the ``/backfill`` API. -To request more history, the requesting homeserver picks another homeserver -that it thinks may have more (most likely this should be a homeserver for some -of the existing users in the room at the earliest point in history it has -currently), and makes a ``/backfill`` request. The parameters of this request -give an event ID that the requesting homeserver wishes to obtain, and a number -specifying how many more events of history before that one to return at most. - -The response to this request is an object with the following keys: - -======================== ============ ========================================= - Key Type Description -======================== ============ ========================================= -``pdus`` List A list of events. -``origin`` String The name of the resident homeserver. -``origin_server_ts`` Integer A timestamp added by the resident - homeserver. -======================== ============ ========================================= - -The list of events given in ``pdus`` is returned in reverse chronological -order; having the most recent event first (i.e. the event whose event ID is -that requested by the requester in the ``v`` parameter). +To request more history, the requesting homeserver picks another homeserver +that it thinks may have more (most likely this should be a homeserver for +some of the existing users in the room at the earliest point in history it +has currently), and makes a ``/backfill`` request. .. TODO-spec Specify (or remark that it is unspecified) how the server handles divergent history. DFS? BFS? Anything weirder? +{{backfill_ss_http_api}} + Inviting to a room ------------------ From 4b1955f395b025edcdcd7ff321927a48f2e082d0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 2 Aug 2018 12:06:43 -0600 Subject: [PATCH 266/308] Clarify what the auth_events on a PDU are --- api/server-server/definitions/unsigned_pdu.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml index ab6e5b1c..f042dffb 100644 --- a/api/server-server/definitions/unsigned_pdu.yaml +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -82,7 +82,9 @@ properties: example: 12 auth_events: type: array - description: Event IDs and hashes for the "auth events" of this event. + description: |- + An event reference list containing the authorization events that would + allow this event to be in the room. items: type: array maxItems: 2 From 9d474bb8195b3a61b1cad0330aa3f1adec7756e7 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 2 Aug 2018 12:44:54 -0600 Subject: [PATCH 267/308] Document event retrieval endpoints in more detail This also adds a previously-undocumented endpoint: /state_ids Backfill is technically not part of this section, however it is being left untouched to make the merge with #1469 easier (which moves it out of the file). Reference material: * Some calls to synapse on these endpoints with a relatively simple private room. --- api/server-server/events.yaml | 76 +++++++++++++++++++++++++++-- specification/server_server_api.rst | 12 ++++- 2 files changed, 83 insertions(+), 5 deletions(-) diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml index d540085d..f0c3250c 100644 --- a/api/server-server/events.yaml +++ b/api/server-server/events.yaml @@ -27,7 +27,7 @@ paths: get: summary: Get all the state of a given room description: |- - Retrieves a snapshot of the entire current state of the given room. + Retrieves a snapshot of a room's state at a given event. operationId: getRoomState parameters: - in: path @@ -36,11 +36,81 @@ paths: description: The room ID to get state for. required: true x-example: "!abc123:matrix.org" + - in: query + name: event_id + type: string + description: An event ID in the room to retrieve the state at. + required: true + x-example: "$helloworld:matrix.org" responses: 200: - description: The room state for the room (kept under ``pdus``). + description: |- + The fully resolved state for the room, including the authorization + chain for the events. schema: - $ref: "definitions/transaction.yaml" + type: object + properties: + auth_chain: + type: list + description: |- + The full set of authorization events that make up the state + of the room, and their authorization events, recursively. + items: + $ref: "definitions/pdu.yaml" + example: [{"$ref": "examples/pdu.json"}] + pdus: + type: list + description: |- + The fully resolved state of the room at the given event. + items: + $ref: "definitions/pdu.yaml" + example: [{"$ref": "examples/pdu.json"}] + required: ['auth_chain', 'pdus'] + "/state_ids/{roomId}": + get: + summary: Get all the state event IDs of a given room + description: |- + Retrieves a snapshot of a room's state at a given event, in the form of + event IDs. This performs the same function as calling ``/state/{roomId}``, + however this returns just the event IDs rather than the full events. + operationId: getRoomStateIds + parameters: + - in: path + name: roomId + type: string + description: The room ID to get state for. + required: true + x-example: "!abc123:matrix.org" + - in: query + name: event_id + type: string + description: An event ID in the room to retrieve the state at. + required: true + x-example: "$helloworld:matrix.org" + responses: + 200: + description: |- + The fully resolved state for the room, including the authorization + chain for the events. + schema: + type: object + properties: + auth_chain_ids: + type: list + description: |- + The full set of authorization events that make up the state + of the room, and their authorization events, recursively. + items: + type: string + example: ["$an_event:domain.com"] + pdu_ids: + type: list + description: |- + The fully resolved state of the room at the given event. + items: + type: string + example: ["$an_event:domain.com"] + required: ['auth_chain_ids', 'pdu_ids'] "/event/{eventId}": get: summary: Get a single event diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 3bdc5b2b..4a00db98 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -606,8 +606,6 @@ All these URLs are name-spaced within a prefix of:: {{transactions_ss_http_api}} -{{events_ss_http_api}} - {{query_general_ss_http_api}} @@ -798,6 +796,16 @@ that requested by the requester in the ``v`` parameter). Specify (or remark that it is unspecified) how the server handles divergent history. DFS? BFS? Anything weirder? +Retriving events +---------------- + +In some circumstances, a homeserver may be missing a particular event or information +about the room which cannot be easily determined from backfilling. These APIs provide +homeservers with the option of getting events and the state of the room at a given +point in the timeline. + +{{events_ss_http_api}} + Inviting to a room ------------------ From 8b7bc603670c2e81c73505b0987690b6b87fa269 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 2 Aug 2018 16:42:03 -0600 Subject: [PATCH 268/308] list -> array --- api/server-server/events.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/server-server/events.yaml b/api/server-server/events.yaml index f0c3250c..11c88e8d 100644 --- a/api/server-server/events.yaml +++ b/api/server-server/events.yaml @@ -51,7 +51,7 @@ paths: type: object properties: auth_chain: - type: list + type: array description: |- The full set of authorization events that make up the state of the room, and their authorization events, recursively. @@ -59,7 +59,7 @@ paths: $ref: "definitions/pdu.yaml" example: [{"$ref": "examples/pdu.json"}] pdus: - type: list + type: array description: |- The fully resolved state of the room at the given event. items: @@ -96,7 +96,7 @@ paths: type: object properties: auth_chain_ids: - type: list + type: array description: |- The full set of authorization events that make up the state of the room, and their authorization events, recursively. @@ -104,7 +104,7 @@ paths: type: string example: ["$an_event:domain.com"] pdu_ids: - type: list + type: array description: |- The fully resolved state of the room at the given event. items: From 5596243add762c0ee2ede0a5ca592f4437ee3ce2 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 2 Aug 2018 16:43:24 -0600 Subject: [PATCH 269/308] origin is required --- api/server-server/joins.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server-server/joins.yaml b/api/server-server/joins.yaml index 472d63bf..759361b7 100644 --- a/api/server-server/joins.yaml +++ b/api/server-server/joins.yaml @@ -273,7 +273,7 @@ paths: type: object schema: $ref: "definitions/pdu.yaml" - required: ["auth_chain", "state"] + required: ["auth_chain", "state", "origin"] examples: application/json: [ 200, From a9258ed1951d55e19f4ede02b7708e692a1bb1f0 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 2 Aug 2018 16:43:29 -0600 Subject: [PATCH 270/308] an -> a --- specification/server_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index d819b072..8c3d85ea 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -759,7 +759,7 @@ and ``event_id`` on the templated event received by the resident server. This event is then signed by the joining server. To complete the join handshake, the joining server must now submit this new -event to an resident homeserver, by using the ``PUT /send_join`` endpoint. +event to a resident homeserver, by using the ``PUT /send_join`` endpoint. The resident homeserver then accepts this event into the room's event graph, and responds to the joining server with the full set of state for the From 05bb7e105015fc0a07ba7e0b2506f49502fd6ec4 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 2 Aug 2018 18:45:59 -0600 Subject: [PATCH 271/308] Spelling and word choice --- api/server-server/query.yaml | 6 +++--- specification/server_server_api.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/server-server/query.yaml b/api/server-server/query.yaml index 4599d130..2f4b8d1e 100644 --- a/api/server-server/query.yaml +++ b/api/server-server/query.yaml @@ -52,7 +52,7 @@ paths: that belong to the target server (idenfified by the DNS Name in the alias). The target server may not appear in the resident servers list. - Servers may wish to cache the response to this query to prevent requesting the + Servers may wish to cache the response to this query to avoid requesting the information too often. operationId: queryRoomDirectory parameters: @@ -76,7 +76,7 @@ paths: servers: type: array description: |- - An array of server names that are likely to hold then given room. This + An array of server names that are likely to hold the given room. This list may or may not include the server answering the query. items: type: string @@ -120,7 +120,7 @@ paths: for a given user. Homeservers should only query profiles for users that belong to the target server (identified by the DNS Name in the user ID). - Servers may wish to cache the response to this query to prevent requesting the + Servers may wish to cache the response to this query to avoid requesting the information too often. parameters: - in: query diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index bfc98fb8..36591356 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -968,7 +968,7 @@ Rejecting a presence invite:: Querying for information ------------------------ -Queries are a way to retrieve information from a homeserver abotu a resource, +Queries are a way to retrieve information from a homeserver about a resource, such as a user or room. The endpoints here are often called in conjunction with a request from a client on the client-server API in order to complete the call. From 0b313dbdd9d1a2f098db4240a98429b9ec01c686 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 2 Aug 2018 18:47:17 -0600 Subject: [PATCH 272/308] Don't explain what a 400 Bad Request is --- api/server-server/query.yaml | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/api/server-server/query.yaml b/api/server-server/query.yaml index 2f4b8d1e..f569549e 100644 --- a/api/server-server/query.yaml +++ b/api/server-server/query.yaml @@ -49,8 +49,7 @@ paths: description: |- Performs a query to get the mapped room ID and list of resident homeservers in the room for a given room alias. Homeservers should only query room aliases - that belong to the target server (idenfified by the DNS Name in the alias). - The target server may not appear in the resident servers list. + that belong to the target server (identified by the DNS Name in the alias). Servers may wish to cache the response to this query to avoid requesting the information too often. @@ -92,17 +91,6 @@ paths: "another.example.com:8449", ] } - 400: - description: |- - The room alias is not hosted on the server. This can happen if the directory - server is named "example.org" and the room alias ends with "matrix.org". - schema: - $ref: "../client-server/definitions/errors/error.yaml" - examples: - application/json: { - "errcode": "M_UNKNOWN", - "error": "Room alias not hosted on this homeserver." - } 404: description: The room alias was not found. schema: @@ -167,21 +155,6 @@ paths: "displayname": "John Doe", "avatar_url": "mxc://matrix.org/MyC00lAvatar" } - 400: - description: |- - The request was missing parameters or had invalid values for the parameters. This - can happen for: - - * The user not being hosted on the homeserver, - * Lack of a ``user_id`` in the request, or - * The ``field`` requested not being an allowed value. - schema: - $ref: "../client-server/definitions/errors/error.yaml" - examples: - application/json: { - "errcode": "M_INVALID_ARGUMENT_VALUE", - "error": "User is not hosted on this homeserver." - } 404: description: The user does not exist or does not have a profile. schema: From e766606f24820f75820b1ae45c7b1a729d64ddfb Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 2 Aug 2018 18:54:12 -0600 Subject: [PATCH 273/308] join -> leave --- api/server-server/leaving.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server-server/leaving.yaml b/api/server-server/leaving.yaml index 11b0e01d..e287bf58 100644 --- a/api/server-server/leaving.yaml +++ b/api/server-server/leaving.yaml @@ -25,7 +25,7 @@ produces: paths: "/make_leave/{roomId}/{userId}": get: - summary: Get information required to make a join event for a room + summary: Get information required to make a leave event for a room description: |- Asks the receiving server to return information that the sending server will need to prepare a leave event to get out of the room. From cda88f3b3debfcacb4ea2ec37ed8309d3090dd8e Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 2 Aug 2018 23:18:08 -0600 Subject: [PATCH 274/308] Document /get_missing_events Fixes https://github.com/matrix-org/matrix-doc/issues/1385 --- api/server-server/backfill.yaml | 66 ++++++++++++++++++++++++++++- specification/server_server_api.rst | 8 +++- 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/api/server-server/backfill.yaml b/api/server-server/backfill.yaml index f83057ae..072b28d1 100644 --- a/api/server-server/backfill.yaml +++ b/api/server-server/backfill.yaml @@ -27,7 +27,7 @@ paths: get: summary: Retrieves the events which precede the given event description: |- - Retreives a sliding-window history of previous PDUs that occurred in the given room. + Retrieves a sliding-window history of previous PDUs that occurred in the given room. Starting from the PDU ID(s) given in the ``v`` argument, the PDUs that preceded it are retrieved, up to the total number given by the ``limit``. operationId: backfillRoom @@ -75,3 +75,67 @@ paths: }, ] } + "/get_missing_events/{roomId}": + post: + summary: Retrieves events that the sender is missing + description: |- + Retrieves the previous events that the sender is missing for a given set of + events. + operationId: getMissingPreviousEvents + parameters: + - in: path + name: roomId + type: string + description: The room ID to search in. + required: true + x-example: "!SomeRoom:matrix.org" + - in: body + name: body + schema: + type: object + properties: + limit: + type: integer + description: The maximum number of events to retrieve. Defaults to 10. + example: 10 + min_depth: + type: integer + description: The minimum depth of events to retrieve. Defaults to 0. + example: 0 + earliest_events: + type: array + description: |- + The events the sender is missing. The response will only contain events + that didn't happen before these events. + items: + type: string + example: ["$missing_event:domain.com"] + latest_events: + type: array + description: |- + The events that the sender has, but does not have previous events for. + items: + type: string + example: ["$event_that_has_the_missing_event_as_a_previous_event:domain.com"] + required: ['earliest_events', 'latest_events'] + responses: + 200: + description: |- + The events that happened after the ``earliest_events`` which are also previous + to the ``latest_events`` and have the minimum depth specified. Does not include + any of the events in the request. + schema: + type: object + properties: + events: + type: array + description: The missing events. + items: + $ref: definitions/pdu.yaml + required: ['events'] + examples: + application/json: { + "events": [ + {"$ref": "examples/pdu.json"} + ] + } diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 090885bc..d7a22e72 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -836,8 +836,8 @@ following keys: here. What purpose does it serve expanding them out in full, when surely they'll appear in the state anyway? -Backfilling ------------ +Backfilling and retrieving missing events +----------------------------------------- Once a homeserver has joined a room, it receives all the events emitted by other homeservers in that room, and is thus aware of the entire history of the @@ -855,6 +855,10 @@ that it thinks may have more (most likely this should be a homeserver for some of the existing users in the room at the earliest point in history it has currently), and makes a ``/backfill`` request. +Similar to backfilling a room's history, a server may not have all the events +in the graph. That server may use the ``/get_missing_events`` API to acquire +the events it is missing. + .. TODO-spec Specify (or remark that it is unspecified) how the server handles divergent history. DFS? BFS? Anything weirder? From 73958ecbffbf1ea996961e92d78e354ceb914200 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 2 Aug 2018 16:41:24 -0600 Subject: [PATCH 275/308] Document /event_auth and /query_auth /event_auth is a fairly easy endpoint to determine the use case of. /query_auth is a little harder to investigate and has a fairly interesting purpose: it appears to be used for the sending server to admit defeat and shop around for the right auth chain, correcting it's own perspective as it goes. /query_auth is based off the following research points in synapse: * https://github.com/matrix-org/synapse/blob/43ecfe0b1028fea5e4dda197f5631aed67182ee6/synapse/handlers/federation.py#L1947-L1990 * https://github.com/matrix-org/synapse/blob/43ecfe0b1028fea5e4dda197f5631aed67182ee6/synapse/handlers/federation.py#L2049-L2187 * https://github.com/matrix-org/synapse/blob/43ecfe0b1028fea5e4dda197f5631aed67182ee6/synapse/handlers/federation.py#L1716-L1761 * https://github.com/matrix-org/synapse/blob/43ecfe0b1028fea5e4dda197f5631aed67182ee6/synapse/federation/federation_server.py#L393-L446 * https://github.com/matrix-org/synapse/blob/master/synapse/federation/transport/server.py#L482-L487 --- api/server-server/event_auth.yaml | 172 ++++++++++++++++++++++++++++ specification/server_server_api.rst | 9 ++ 2 files changed, 181 insertions(+) create mode 100644 api/server-server/event_auth.yaml diff --git a/api/server-server/event_auth.yaml b/api/server-server/event_auth.yaml new file mode 100644 index 00000000..77d6fb1e --- /dev/null +++ b/api/server-server/event_auth.yaml @@ -0,0 +1,172 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Event Authorization API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/event_auth/{roomId}/{eventId}": + get: + summary: Get the auth chain for a given event + description: |- + Retrieves the complete auth chain for a given event. + operationId: getEventAuth + parameters: + - in: path + name: roomId + type: string + description: The room ID to get the auth chain for. + required: true + x-example: "!abc123:matrix.org" + - in: path + name: eventId + type: string + description: The event ID to get the auth chain of. + required: true + x-example: "$helloworld:domain.com" + responses: + 200: + description: The auth chain for the event. + schema: + type: object + properties: + auth_chain: + type: array + description: |- + The full set of authorization events that make up the state of + the room, and their authorization events, recursively. + items: + $ref: "definitions/pdu.yaml" + example: [{"$ref": "examples/pdu.json"}] + required: ['auth_chain'] + "/query_auth/{roomId}/{eventId}": + post: + summary: Compare auth chains with the receiving server + description: |- + Compares the auth chain provided with what the receiving server has for the + room ID and event ID combination. + + The auth difference can be calculated in two parts, where the "remote auth" + is the auth chain provided by the sending server and the "local auth" is the + auth chain the receiving server has. With those lists, the algorithm works + bottom-up after sorting each chain by depth then by event ID. The differences + are then discovered and returned as the response to this API call. + operationId: compareEventAuth + parameters: + - in: path + name: roomId + type: string + description: The room ID to compare the auth chain in. + required: true + x-example: "!abc123:matrix.org" + - in: path + name: eventId + type: string + description: The event ID to compare the auth chain of. + required: true + x-example: "$helloworld:domain.com" + - in: body + name: body + schema: + type: object + properties: + auth_chain: + type: array + description: The auth chain (the "remote auth"). + items: + $ref: "definitions/pdu.yaml" + example: [{"$ref": "examples/pdu.json"}] + missing: + type: array + description: |- + A list of event IDs that the sender thinks the receiver is missing. + items: + type: string + example: [] + rejects: + type: object + description: |- + The set of events that the sending server has rejected from the provided + auth chain. + + The ``string`` key is the event ID that was rejected. + additionalProperties: + type: object + title: Rejection Reason + properties: + reason: + type: enum + enum: ['auth_error', 'replaced', 'not_ancestor'] + description: |- + The reason for the event being rejected. + required: ['reason'] + example: { + "$some_event:domain.com": { + "reason": "auth_error" + } + } + required: ['auth_chain'] + responses: + 200: + description: The auth chain differences, as determined by the receiver. + schema: + type: object + properties: + auth_chain: + type: array + description: |- + The auth chain the receiver has, and used to determine the auth + chain differences (the "local auth"). + items: + $ref: "definitions/pdu.yaml" + example: [{"$ref": "examples/pdu.json"}] + missing: + type: array + description: |- + The list of event IDs that the receiver believes it is missing, + after comparing the "remote auth" and "local auth" chains. + items: + type: string + example: ["$a_missing_event:domain.com"] + rejects: + type: object + description: |- + The set of events that the receiving server has rejected from the + auth chain, not including events that the sending server is missing + as determined from the difference algorithm. + + The ``string`` key is the event ID that was rejected. + additionalProperties: + type: object + title: Rejection Reason + properties: + reason: + type: enum + enum: ['auth_error', 'replaced', 'not_ancestor'] + description: |- + The reason for the event being rejected. + required: ['reason'] + example: { + "$some_event:domain.com": { + "reason": "auth_error" + } + } + required: ['auth_chain', 'missing', 'rejects'] diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 01172c6e..bded4a49 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -440,6 +440,15 @@ The rules are as follows: I think there is some magic about 3pid invites too. +Retrieving event authorization information +++++++++++++++++++++++++++++++++++++++++++ + +The homeserver may be missing event authorization information, or wish to check +with other servers to ensure it is receiving the correct auth chain. These APIs +give the homeserver an avenue for getting the information it needs. + +{{event_auth_ss_http_api}} + EDUs ---- From 2ac80d38d7b38653bb0e97dfbe6f272fb902a03b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 3 Aug 2018 11:23:44 -0600 Subject: [PATCH 276/308] Document the /publicRooms endpoint for federation This intentionally doesn't document the third party network aspect of the endpoint. This is scheduled for a later area for dealing with third party network/IDs and is reported as https://github.com/matrix-org/matrix-doc/issues/1476 The client-server response has been broken out to a shared file: both the client-server and server-server /publicRoom endpoints return the same thing, with slightly different inputs. The inputs (and behaviour) are based upon the docstring here: https://github.com/matrix-org/synapse/blob/43ecfe0b1028fea5e4dda197f5631aed67182ee6/synapse/federation/transport/server.py#L583-L612 --- .../definitions/public_rooms_response.yaml | 105 ++++++++++++++++++ api/client-server/list_public_rooms.yaml | 93 +--------------- api/server-server/public_rooms.yaml | 52 +++++++++ specification/server_server_api.rst | 10 ++ 4 files changed, 168 insertions(+), 92 deletions(-) create mode 100644 api/client-server/definitions/public_rooms_response.yaml create mode 100644 api/server-server/public_rooms.yaml diff --git a/api/client-server/definitions/public_rooms_response.yaml b/api/client-server/definitions/public_rooms_response.yaml new file mode 100644 index 00000000..fc6ccb44 --- /dev/null +++ b/api/client-server/definitions/public_rooms_response.yaml @@ -0,0 +1,105 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +type: object +description: A list of the rooms on the server. +required: ["chunk"] +properties: + chunk: + title: "PublicRoomsChunks" + type: array + description: |- + A paginated chunk of public rooms. + items: + type: object + title: "PublicRoomsChunk" + required: + - room_id + - num_joined_members + - world_readable + - guest_can_join + properties: + aliases: + type: array + description: |- + Aliases of the room. May be empty. + items: + type: string + canonical_alias: + type: string + description: |- + The canonical alias of the room, if any. + name: + type: string + description: |- + The name of the room, if any. + num_joined_members: + type: number + description: |- + The number of members joined to the room. + room_id: + type: string + description: |- + The ID of the room. + topic: + type: string + description: |- + The topic of the room, if any. + world_readable: + type: boolean + description: |- + Whether the room may be viewed by guest users without joining. + guest_can_join: + type: boolean + description: |- + Whether guest users may join the room and participate in it. + If they can, they will be subject to ordinary power level + rules like any other user. + avatar_url: + type: string + description: The URL for the room's avatar, if one is set. + next_batch: + type: string + description: |- + A pagination token for the response. The absence of this token + means there are no more results to fetch and the client should + stop paginating. + prev_batch: + type: string + description: |- + A pagination token that allows fetching previous results. The + absence of this token means there are no results before this + batch, i.e. this is the first batch. + total_room_count_estimate: + type: number + description: |- + An estimate on the total number of public rooms, if the + server has an estimate. +example: { + "chunk": [ + { + "aliases": ["#murrays:cheese.bar"], + "avatar_url": "mxc://bleeker.street/CHEDDARandBRIE", + "guest_can_join": false, + "name": "CHEESE", + "num_joined_members": 37, + "room_id": "!ol19s:bleecker.street", + "topic": "Tasty tasty cheese", + "world_readable": true + } + ], + "next_batch": "p190q", + "prev_batch": "p1902", + "total_room_count_estimate": 115 +} \ No newline at end of file diff --git a/api/client-server/list_public_rooms.yaml b/api/client-server/list_public_rooms.yaml index 45034488..72a12060 100644 --- a/api/client-server/list_public_rooms.yaml +++ b/api/client-server/list_public_rooms.yaml @@ -144,98 +144,7 @@ paths: 200: description: A list of the rooms on the server. schema: - type: object - description: A list of the rooms on the server. - required: ["chunk"] - properties: - chunk: - title: "PublicRoomsChunks" - type: array - description: |- - A paginated chunk of public rooms. - items: - type: object - title: "PublicRoomsChunk" - required: - - room_id - - num_joined_members - - world_readable - - guest_can_join - properties: - aliases: - type: array - description: |- - Aliases of the room. May be empty. - items: - type: string - canonical_alias: - type: string - description: |- - The canonical alias of the room, if any. - name: - type: string - description: |- - The name of the room, if any. - num_joined_members: - type: number - description: |- - The number of members joined to the room. - room_id: - type: string - description: |- - The ID of the room. - topic: - type: string - description: |- - The topic of the room, if any. - world_readable: - type: boolean - description: |- - Whether the room may be viewed by guest users without joining. - guest_can_join: - type: boolean - description: |- - Whether guest users may join the room and participate in it. - If they can, they will be subject to ordinary power level - rules like any other user. - avatar_url: - type: string - description: The URL for the room's avatar, if one is set. - next_batch: - type: string - description: |- - A pagination token for the response. The absence of this token - means there are no more results to fetch and the client should - stop paginating. - prev_batch: - type: string - description: |- - A pagination token that allows fetching previous results. The - absence of this token means there are no results before this - batch, i.e. this is the first batch. - total_room_count_estimate: - type: number - description: |- - An estimate on the total number of public rooms, if the - server has an estimate. - examples: - application/json: { - "chunk": [ - { - "aliases": ["#murrays:cheese.bar"], - "avatar_url": "mxc://bleeker.street/CHEDDARandBRIE", - "guest_can_join": false, - "name": "CHEESE", - "num_joined_members": 37, - "room_id": "!ol19s:bleecker.street", - "topic": "Tasty tasty cheese", - "world_readable": true - } - ], - "next_batch": "p190q", - "prev_batch": "p1902", - "total_room_count_estimate": 115 - } + $ref: "definitions/public_rooms_response.yaml" tags: - Room discovery post: diff --git a/api/server-server/public_rooms.yaml b/api/server-server/public_rooms.yaml new file mode 100644 index 00000000..6cd07449 --- /dev/null +++ b/api/server-server/public_rooms.yaml @@ -0,0 +1,52 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swagger: '2.0' +info: + title: "Matrix Federation Public Rooms API" + version: "1.0.0" +host: localhost:8448 +schemes: + - https +basePath: /_matrix/federation/v1 +produces: + - application/json +paths: + "/publicRooms": + get: + summary: Get all the public rooms for a homeserver + description: |- + Gets all the public rooms for the homeserver. This should not return + rooms that are listed on another homeserver's directory, just those + listed on the receiving homeserver's directory. + operationId: getPublicRooms + parameters: + - in: query + name: limit + type: integer + description: |- + The maximum number of rooms to return. Defaults to 0 (no limit). + x-example: 10 + - in: query + name: since + type: string + description: |- + A pagination token from a previous call to this endpoint to fetch more + rooms. + x-example: "GetMoreRoomsTokenHere" + responses: + 200: + description: The public room list for the homeserver. + schema: + $ref: "../client-server/definitions/public_rooms_response.yaml" diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 01172c6e..e058ca68 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -893,6 +893,16 @@ A homeserver may provide a TLS client certificate and the receiving homeserver may check that the client certificate matches the certificate of the origin homeserver. +Public Room Directory +--------------------- + +To compliment the `Client-Server API`_'s room directory, homeservers need a +way to query the public rooms for another server. This can be done by making +a request to the ``/publicRooms`` endpoint for the server the room directory +should be retrieved for. + +{{public_rooms_ss_http_api}} + Presence -------- From 96f551026695ac0e9a5aa16a960cf80e6fbd72ae Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 3 Aug 2018 12:03:23 -0600 Subject: [PATCH 277/308] Ensure the example is always set to something --- scripts/templating/matrix_templates/units.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index d6579e6c..045826f2 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -681,6 +681,7 @@ class MatrixUnits(Units): try: example = get_example_for_schema(definition) except: + example = None pass # do nothing - we don't care if 'title' not in definition: definition['title'] = "NO_TITLE" From 5fbaa1deb564fa39728e1e9628f92a976a7d6188 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 3 Aug 2018 13:10:24 -0600 Subject: [PATCH 278/308] Federation: Document the Host header; Clarify how literal IP addresses are handled Fixes https://github.com/matrix-org/matrix-doc/issues/1161 The IP address clarification is to add an explicit mention of how to handle the case. The prior documentation assumed that all servers would be resolvable with DNS, and does technically have a fallback to use the fallback port, however making it clear feels like a good idea. --- specification/server_server_api.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 01172c6e..b2ff9c6e 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -98,6 +98,15 @@ default fallback port number of 8448. Homeservers may use SRV records to load balance requests between multiple TLS endpoints or to failover to another endpoint if an endpoint fails. +If the DNS name is a literal IP address, the port specified or the fallback +port should be used. + +When making requests to servers, use the DNS name of the target server in the +``Host`` header, regardless of the host given in the SRV record. For example, +if making a request to ``example.org``, and the SRV record resolves to ``matrix. +example.org``, the ``Host`` header in the request should be ``example.org``. The +port number for target server should not appear in the ``Host`` header. + Server implementation ~~~~~~~~~~~~~~~~~~~~~~ From 6ac25d167c192d3be5a081170c2d67703cf02bdf Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 3 Aug 2018 13:57:13 -0600 Subject: [PATCH 279/308] get_missing_events consumes JSON --- api/server-server/backfill.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/server-server/backfill.yaml b/api/server-server/backfill.yaml index 072b28d1..d4dad2ec 100644 --- a/api/server-server/backfill.yaml +++ b/api/server-server/backfill.yaml @@ -20,6 +20,8 @@ host: localhost:8448 schemes: - https basePath: /_matrix/federation/v1 +consumes: + - application/json produces: - application/json paths: From 13a2b54bf934c30b84b7caf7c1c3d003dbe9a7b2 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 3 Aug 2018 13:57:43 -0600 Subject: [PATCH 280/308] query_auth consumes JSON --- api/server-server/event_auth.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/server-server/event_auth.yaml b/api/server-server/event_auth.yaml index 77d6fb1e..f55afddc 100644 --- a/api/server-server/event_auth.yaml +++ b/api/server-server/event_auth.yaml @@ -20,6 +20,8 @@ host: localhost:8448 schemes: - https basePath: /_matrix/federation/v1 +consumes: + - application/json produces: - application/json paths: From 8dd63388e0a9646bf8904ce14f40bd7635a2092a Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 3 Aug 2018 13:58:27 -0600 Subject: [PATCH 281/308] invites consume JSON --- api/server-server/invites.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/server-server/invites.yaml b/api/server-server/invites.yaml index d1838bf5..98d53d49 100644 --- a/api/server-server/invites.yaml +++ b/api/server-server/invites.yaml @@ -20,6 +20,8 @@ host: localhost:8448 schemes: - https basePath: /_matrix/federation/v1 +consumes: + - application/json produces: - application/json paths: From 2e6d6e27908a31e54e4bb411dbf58ba01e52495b Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 3 Aug 2018 14:01:33 -0600 Subject: [PATCH 282/308] Specify which swagger APIs consume JSON --- api/server-server/joins.yaml | 2 ++ api/server-server/keys_query.yaml | 2 ++ api/server-server/leaving.yaml | 2 ++ api/server-server/third_party_invite.yaml | 2 ++ api/server-server/transactions.yaml | 2 ++ 5 files changed, 10 insertions(+) diff --git a/api/server-server/joins.yaml b/api/server-server/joins.yaml index 759361b7..14142945 100644 --- a/api/server-server/joins.yaml +++ b/api/server-server/joins.yaml @@ -20,6 +20,8 @@ host: localhost:8448 schemes: - https basePath: /_matrix/federation/v1 +consumes: + - application/json produces: - application/json paths: diff --git a/api/server-server/keys_query.yaml b/api/server-server/keys_query.yaml index 8fbe00dc..e616915b 100644 --- a/api/server-server/keys_query.yaml +++ b/api/server-server/keys_query.yaml @@ -20,6 +20,8 @@ host: localhost:8448 schemes: - https basePath: /_matrix/key/v2 +consumes: + - application/json produces: - application/json paths: diff --git a/api/server-server/leaving.yaml b/api/server-server/leaving.yaml index e287bf58..28bcf42c 100644 --- a/api/server-server/leaving.yaml +++ b/api/server-server/leaving.yaml @@ -20,6 +20,8 @@ host: localhost:8448 schemes: - https basePath: /_matrix/federation/v1 +consumes: + - application/json produces: - application/json paths: diff --git a/api/server-server/third_party_invite.yaml b/api/server-server/third_party_invite.yaml index 42bdae23..754a3282 100644 --- a/api/server-server/third_party_invite.yaml +++ b/api/server-server/third_party_invite.yaml @@ -20,6 +20,8 @@ host: localhost:8448 schemes: - https basePath: /_matrix/federation/v1 +consumes: + - application/json produces: - application/json paths: diff --git a/api/server-server/transactions.yaml b/api/server-server/transactions.yaml index 2a9180a1..4f4c6b28 100644 --- a/api/server-server/transactions.yaml +++ b/api/server-server/transactions.yaml @@ -20,6 +20,8 @@ host: localhost:8448 schemes: - https basePath: /_matrix/federation/v1 +consumes: + - application/json produces: - application/json paths: From ef9d766dc3551fd23b0410dab9d274a599ab0803 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 3 Aug 2018 14:51:46 -0600 Subject: [PATCH 283/308] Remove irrelevant TODO We fixed the EDU, so we don't need this comment. --- api/server-server/definitions/edu.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/server-server/definitions/edu.yaml b/api/server-server/definitions/edu.yaml index bc1f0cce..725af695 100644 --- a/api/server-server/definitions/edu.yaml +++ b/api/server-server/definitions/edu.yaml @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO: Address any concerns about this being inaccurate (flagged as such in the EDUs section) - type: object title: Ephemeral Data Unit description: An ephemeral data unit. From db6d466fa1db004afc9ea7909224bc794d2e4b51 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 3 Aug 2018 19:49:46 -0600 Subject: [PATCH 284/308] Go one level deep when finding definitions This is useful for when we start defining event schemas. This also has a sanity check for ensuring the directory exists, allowing the IS and push API paths to be uncommented. --- scripts/templating/matrix_templates/units.py | 66 ++++++++++++-------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 045826f2..88f7b86c 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -46,8 +46,8 @@ HTTP_APIS = { SWAGGER_DEFINITIONS = { os.path.join(matrix_doc_dir, "api/application-service/definitions"): "as", os.path.join(matrix_doc_dir, "api/client-server/definitions"): "cs", - #os.path.join(matrix_doc_dir, "api/identity/definitions"): "is", - #os.path.join(matrix_doc_dir, "api/push-gateway/definitions"): "push", + os.path.join(matrix_doc_dir, "api/identity/definitions"): "is", + os.path.join(matrix_doc_dir, "api/push-gateway/definitions"): "push", os.path.join(matrix_doc_dir, "api/server-server/definitions"): "ss", } EVENT_EXAMPLES = os.path.join(matrix_doc_dir, "event-schemas/examples") @@ -665,33 +665,45 @@ class MatrixUnits(Units): def load_swagger_definitions(self): defs = {} for path, prefix in SWAGGER_DEFINITIONS.items(): - for filename in os.listdir(path): - if not filename.endswith(".yaml"): - continue - filepath = os.path.join(path, filename) - logger.info("Reading swagger definition: %s" % filepath) - with open(filepath, "r") as f: - # strip .yaml - group_name = filename[:-5].replace("-", "_") - group_name = "%s_%s" % (prefix, group_name) - definition = yaml.load(f.read(), OrderedLoader) - definition = resolve_references(filepath, definition) - if 'type' not in definition: - continue - try: - example = get_example_for_schema(definition) - except: - example = None - pass # do nothing - we don't care - if 'title' not in definition: - definition['title'] = "NO_TITLE" - definition['tables'] = get_tables_for_schema(definition) - defs[group_name] = { - "definition": definition, - "examples": [example] if example is not None else [], - } + self._load_swagger_definitions_in_dir(defs, path, prefix) return defs + def _load_swagger_definitions_in_dir(self, defs, path, prefix, recurse=True): + if not os.path.exists(path): + return defs + for filename in os.listdir(path): + filepath = os.path.join(path, filename) + if os.path.isdir(filepath) and recurse: + safe_name = re.sub(r"[^a-zA-Z0-9_]", "_", filename) + dir_prefix = "_".join([prefix, safe_name]) + # We don't recurse because we have to stop at some point + self._load_swagger_definitions_in_dir( + defs, filepath, dir_prefix, recurse=False) + if not filename.endswith(".yaml"): + continue + filepath = os.path.join(path, filename) + logger.info("Reading swagger definition: %s" % filepath) + with open(filepath, "r") as f: + # strip .yaml + group_name = re.sub(r"[^a-zA-Z0-9_]", "_", filename[:-5]) + group_name = "%s_%s" % (prefix, group_name) + definition = yaml.load(f.read(), OrderedLoader) + definition = resolve_references(filepath, definition) + if 'type' not in definition: + continue + try: + example = get_example_for_schema(definition) + except: + example = None + pass # do nothing - we don't care + if 'title' not in definition: + definition['title'] = "NO_TITLE" + definition['tables'] = get_tables_for_schema(definition) + defs[group_name] = { + "definition": definition, + "examples": [example] if example is not None else [], + } + return defs def load_common_event_fields(self): """Parse the core event schema files From 7a46bdae02b841dfd3f400b4382064a1c178a5b9 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Sun, 5 Aug 2018 14:43:05 +0100 Subject: [PATCH 285/308] Spacing --- specification/server_server_api.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 653d75ed..38b25328 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -139,6 +139,7 @@ certificate fingerprints to validate any connection made to the homeserver. {{keys_server_ss_http_api}} + Querying Keys Through Another Server ++++++++++++++++++++++++++++++++++++ From c826edf23b4ae77e29189a54e9ab3109fa5f3ab3 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Sun, 5 Aug 2018 14:46:42 +0100 Subject: [PATCH 286/308] Remove dns_name from S2S Api --- specification/server_server_api.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 38b25328..30cf45ab 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -84,7 +84,6 @@ and an optional TLS port. .. code:: server_name = hostname [ ":" tls_port] - dns_name = tls_port = *DIGIT .. ** From dcb39f8890af57e62ed37f9e894af4b5af5d9bc3 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Sun, 5 Aug 2018 10:55:55 -0600 Subject: [PATCH 287/308] Fix wording from bad merge --- specification/server_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 30cf45ab..cf3333a1 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -629,7 +629,7 @@ if this optimisation fails. Once the joining server has the room ID and the join candidates, it then needs to obtain enough information about the room to fill in the required fields of -the ``m.room.member`` event. It obtains this by selecting a resident from t +the ``m.room.member`` event. It obtains this by selecting a resident from the candidate list, and using the ``GET /make_join`` endpoint. The resident server will then reply with enough information for the joining server to fill in the event. From 803ef536ba64178b6468084c24aa9cddc7835017 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 6 Aug 2018 15:58:30 +0100 Subject: [PATCH 288/308] Fix grammar in room tag spec Have removed the second clause about how the client interprets them: I was trying to think how to rephrase it but I think in reality it's probably just redundant. --- specification/modules/tags.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/modules/tags.rst b/specification/modules/tags.rst index 8c74c55f..f965c2e8 100644 --- a/specification/modules/tags.rst +++ b/specification/modules/tags.rst @@ -41,7 +41,7 @@ after the rooms with that tag that have an ``order`` key. The name of a tag MUST not exceed 255 bytes. -Tags namespaces are defined in the following way, depending on how the client are expected to interpret them: +The tag namespace is defined as follows: * The namespace ``m.*`` is reserved for tags defined in the Matrix specification. Clients must ignore any tags in this namespace they don't understand. From e7cebf670eb5e8beb45d3f8289687b7de3b1abca Mon Sep 17 00:00:00 2001 From: Florian Jacob Date: Mon, 6 Aug 2018 22:56:28 +0200 Subject: [PATCH 289/308] Fix minor spelling mistake from #1472 Signed-off-by: Florian Jacob --- specification/server_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index cf3333a1..bd370149 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -693,7 +693,7 @@ that requested by the requester in the ``v`` parameter). Specify (or remark that it is unspecified) how the server handles divergent history. DFS? BFS? Anything weirder? -Retriving events +Retrieving events ---------------- In some circumstances, a homeserver may be missing a particular event or information From 7ada91787a9a6be86632566911d29d7145fbda5c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 7 Aug 2018 14:20:46 -0600 Subject: [PATCH 290/308] age_ts isn't real --- api/server-server/definitions/unsigned_pdu.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/api/server-server/definitions/unsigned_pdu.yaml b/api/server-server/definitions/unsigned_pdu.yaml index f042dffb..ab281224 100644 --- a/api/server-server/definitions/unsigned_pdu.yaml +++ b/api/server-server/definitions/unsigned_pdu.yaml @@ -120,10 +120,6 @@ properties: type: integer description: The number of milliseconds that have passed since this message was sent. example: 4612 - age_ts: - type: integer - format: int64 - description: The POSIX timestamp this message was sent at in milliseconds. replaces_state: type: string description: The event ID of the state event this event replaces. From cfdbee5fc27a4da2c0d1cb96ffd94e31bc561e91 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 7 Aug 2018 14:21:01 -0600 Subject: [PATCH 291/308] EDU origin and destinations aren't required --- api/server-server/definitions/edu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server-server/definitions/edu.yaml b/api/server-server/definitions/edu.yaml index 725af695..142f4b93 100644 --- a/api/server-server/definitions/edu.yaml +++ b/api/server-server/definitions/edu.yaml @@ -33,4 +33,4 @@ properties: content: type: object description: The content of the ephemeral message. -required: ['edu_type', 'origin', 'destination', 'content'] \ No newline at end of file +required: ['edu_type', 'content'] \ No newline at end of file From 0858e7bd23b102ad2d937c542df81fc8ed8de831 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 7 Aug 2018 16:58:14 -0600 Subject: [PATCH 292/308] Mention how the content repository works over federation We might want to consider promoting the media repo to it's own API, and maybe consider calling it the Media Repo rather than Content Repo. Source of information: experience. --- specification/server_server_api.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 84a76639..af314c1e 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -1020,6 +1020,20 @@ the following EDU:: for the user +Content Repository +------------------ + +Attachments to events (images, files, etc) are uploaded to a homeserver via the +Content Repository described in the `Client-Server API`_. When a server wishes +to serve content originating from a remote server, it needs to ask the remote +server for the media. + +Servers should use the server described in the Matrix Content URI, which has the +format ``mxc://{ServerName}/{MediaID}``. Servers should use the download endpoint +described in the `Client-Server API`_, being sure to use the ``allow_remote`` +parameter (set to ``false``). + + Signing Events -------------- From cdcd72c0e3e8212828e0f6ddbf28b84b6cabdd00 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 7 Aug 2018 17:02:24 -0600 Subject: [PATCH 293/308] Fix stray hash on client-server API link This shouldn't be here, and nothing appears to need it. Removing it now because it has been noticed. --- specification/server_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index af314c1e..4f11f917 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -1182,7 +1182,7 @@ that are too long. .. _`Invitation storage`: ../identity_service/unstable.html#invitation-storage .. _`Identity Service API`: ../identity_service/unstable.html -.. _`Client-Server API`: ../client_server/unstable.html#m-room-member +.. _`Client-Server API`: ../client_server/unstable.html .. _`Inviting to a room`: #inviting-to-a-room .. _`Canonical JSON`: ../appendices.html#canonical-json .. _`Unpadded Base64`: ../appendices.html#unpadded-base64 From 57eeddd849e505e19f046f0bff52a57d916330b7 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 8 Aug 2018 07:43:22 -0600 Subject: [PATCH 294/308] Remove unused origin and destination on EDUs --- api/server-server/definitions/edu.yaml | 8 -------- api/server-server/examples/edu.json | 2 -- 2 files changed, 10 deletions(-) diff --git a/api/server-server/definitions/edu.yaml b/api/server-server/definitions/edu.yaml index 142f4b93..0e4edcc6 100644 --- a/api/server-server/definitions/edu.yaml +++ b/api/server-server/definitions/edu.yaml @@ -22,14 +22,6 @@ properties: type: string description: The type of ephemeral message. example: "m.presence" - origin: - type: string - description: The server name sending the ephemeral message. - example: "matrix.org" - destination: - type: string - description: The server name receiving the ephemeral message. - example: "elsewhere.com" content: type: object description: The content of the ephemeral message. diff --git a/api/server-server/examples/edu.json b/api/server-server/examples/edu.json index 9ff40dc3..f5a58e21 100644 --- a/api/server-server/examples/edu.json +++ b/api/server-server/examples/edu.json @@ -1,7 +1,5 @@ { "edu_type": "m.presence", - "origin": "matrix.org", - "destination": "elsewhere.com", "content": { "key": "value" } From 0a7e670715cd31b73bf5bf0c14ebb36f1a76ccb7 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 8 Aug 2018 08:28:52 -0600 Subject: [PATCH 295/308] Document how typing notifications work over federation Relevant synapse code: https://github.com/matrix-org/synapse/blob/d69decd5c78c72abef50b597a689e2bc55a39702/synapse/handlers/typing.py#L221-L230 --- .../definitions/event-schemas/m.typing.yaml | 45 +++++++++++++++++++ specification/server_server_api.rst | 10 +++++ 2 files changed, 55 insertions(+) create mode 100644 api/server-server/definitions/event-schemas/m.typing.yaml diff --git a/api/server-server/definitions/event-schemas/m.typing.yaml b/api/server-server/definitions/event-schemas/m.typing.yaml new file mode 100644 index 00000000..d4fa2f81 --- /dev/null +++ b/api/server-server/definitions/event-schemas/m.typing.yaml @@ -0,0 +1,45 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +type: object +title: Typing Notification EDU +description: A typing notification EDU for a user in a room. +allOf: + - $ref: ../edu.yaml + - type: object + properties: + edu_type: + type: string + description: The string ``m.typing`` + example: "m.typing" + content: + type: object + description: The typing notification. + title: Typing Notification + properties: + room_id: + type: string + description: |- + The room where the user's typing status has been updated. + example: "!somewhere:matrix.org" + user_id: + type: string + description: |- + The user ID that has had their typing status changed. + example: "@john:matrix.org" + typing: + type: boolean + description: Whether the user is typing in the room or not. + example: true + required: ['room_id', 'user_id', 'typing'] diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index a3d3f83a..4188c36b 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -833,6 +833,16 @@ should be retrieved for. {{public_rooms_ss_http_api}} +Typing Notifications +-------------------- + +When a server's users send typing notifications, those notifications need to +be sent to other servers in the room so their users are aware of the same +state. Receiving servers should verify that the user is in the room, and is +a user belonging to the sending server. + +{{definition_ss_event_schemas_m_typing}} + Presence -------- The server API for presence is based entirely on exchange of the following From 721be47eb1070d9e11bb999353c49a4a307f1d6d Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 8 Aug 2018 08:39:59 -0600 Subject: [PATCH 296/308] Describe how get_missing_events actually works The original text wasn't far off, except for being wrong about what the fields actually do. This commit also adds a bit of clarity for how the server is expected to behave. --- api/server-server/backfill.yaml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/api/server-server/backfill.yaml b/api/server-server/backfill.yaml index d4dad2ec..f9f105e2 100644 --- a/api/server-server/backfill.yaml +++ b/api/server-server/backfill.yaml @@ -81,8 +81,9 @@ paths: post: summary: Retrieves events that the sender is missing description: |- - Retrieves the previous events that the sender is missing for a given set of - events. + Retrieves previous events that the sender is missing. This is done by doing a breadth-first + walk of the ``prev_events`` for the ``latest_events``, ignoring any events in ``earliest_events`` + and stopping at the ``limit``. operationId: getMissingPreviousEvents parameters: - in: path @@ -107,15 +108,14 @@ paths: earliest_events: type: array description: |- - The events the sender is missing. The response will only contain events - that didn't happen before these events. + The latest events that the sender already has. These are skipped when retrieving + the previous events of ``latest_events``. items: type: string example: ["$missing_event:domain.com"] latest_events: type: array - description: |- - The events that the sender has, but does not have previous events for. + description: The events to retrieve the previous events for. items: type: string example: ["$event_that_has_the_missing_event_as_a_previous_event:domain.com"] @@ -123,9 +123,8 @@ paths: responses: 200: description: |- - The events that happened after the ``earliest_events`` which are also previous - to the ``latest_events`` and have the minimum depth specified. Does not include - any of the events in the request. + The previous events for ``latest_events``, excluding any ``earliest_events``, up to the + provided ``limit``. schema: type: object properties: From 32178bb07e60117be9377071a407ca4f1f5d6d31 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 9 Aug 2018 08:33:00 -0600 Subject: [PATCH 297/308] Tidy up the specification * Minor word choice changes * Remove the now-empty "Protocol URLs" section --- specification/server_server_api.rst | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 25c9189e..a6d601e6 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -23,14 +23,13 @@ Federation API Matrix homeservers use the Federation APIs (also known as server-server APIs) to communicate with each other. Homeservers use these APIs to push messages to -each other in real-time, to -historic messages from each other, and to +each other in real-time, to retrieve historic messages from each other, and to query profile and presence information about users on each other's servers. -The APIs are implemented using HTTPS GETs and PUTs between each of the -servers. These HTTPS requests are strongly authenticated using public key -signatures at the TLS transport layer and using public key signatures in -HTTP Authorization headers at the HTTP layer. +The APIs are implemented using HTTPS requests between each of the servers. +These HTTPS requests are strongly authenticated using public key signatures +at the TLS transport layer and using public key signatures in HTTP +Authorization headers at the HTTP layer. There are three main kinds of communication that occur between homeservers: @@ -485,15 +484,9 @@ A *conflict* occurs between states where those states have different ``event_ids`` for the same ``(state_type, state_key)``. The events thus affected are said to be *conflicting* events. -Protocol URLs -------------- -.. WARNING:: - This section may be misleading or inaccurate. -All these URLs are name-spaced within a prefix of:: - /_matrix/federation/v1/... Joining Rooms ------------- From 83ea90f7baa838b838489a433bf53f0ca8e11cc5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 3 Aug 2018 11:52:22 -0600 Subject: [PATCH 298/308] Move the Authentication section higher in the spec It forms the foundation for all the requests under it, so it should appear before the endpoints that need it. --- specification/server_server_api.rst | 183 ++++++++++++++-------------- 1 file changed, 91 insertions(+), 92 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index a6d601e6..b045db72 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -162,6 +162,97 @@ multiple servers to mitigate against DNS spoofing. {{keys_query_ss_http_api}} +Authentication +-------------- + +Request Authentication +~~~~~~~~~~~~~~~~~~~~~~ + +Every HTTP request made by a homeserver is authenticated using public key +digital signatures. The request method, target and body are signed by wrapping +them in a JSON object and signing it using the JSON signing algorithm. The +resulting signatures are added as an Authorization header with an auth scheme +of ``X-Matrix``. Note that the target field should include the full path +starting with ``/_matrix/...``, including the ``?`` and any query parameters if +present, but should not include the leading ``https:``, nor the destination +server's hostname. + +Step 1 sign JSON: + +.. code:: + + { + "method": "GET", + "uri": "/target", + "origin": "origin.hs.example.com", + "destination": "destination.hs.example.com", + "content": , + "signatures": { + "origin.hs.example.com": { + "ed25519:key1": "ABCDEF..." + } + } + } + +Step 2 add Authorization header: + +.. code:: + + GET /target HTTP/1.1 + Authorization: X-Matrix origin=origin.example.com,key="ed25519:key1",sig="ABCDEF..." + Content-Type: application/json + + + + +Example python code: + +.. code:: python + + def authorization_headers(origin_name, origin_signing_key, + destination_name, request_method, request_target, + content=None): + request_json = { + "method": request_method, + "uri": request_target, + "origin": origin_name, + "destination": destination_name, + } + + if content_json is not None: + request["content"] = content + + signed_json = sign_json(request_json, origin_name, origin_signing_key) + + authorization_headers = [] + + for key, sig in signed_json["signatures"][origin_name].items(): + authorization_headers.append(bytes( + "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % ( + origin_name, key, sig, + ) + )) + + return ("Authorization", authorization_headers) + +Response Authentication +~~~~~~~~~~~~~~~~~~~~~~~ + +Responses are authenticated by the TLS server certificate. A homeserver should +not send a request until it has authenticated the connected server to avoid +leaking messages to eavesdroppers. + +Client TLS Certificates +~~~~~~~~~~~~~~~~~~~~~~~ + +Requests are authenticated at the HTTP layer rather than at the TLS layer +because HTTP services like Matrix are often deployed behind load balancers that +handle the TLS and these load balancers make it difficult to check TLS client +certificates. + +A homeserver may provide a TLS client certificate and the receiving homeserver +may check that the client certificate matches the certificate of the origin +homeserver. Transactions ------------ @@ -712,98 +803,6 @@ delivered when the invite was stored, this verification will prove that the ``m.room.member`` invite event comes from the user owning the invited third-party identifier. -Authentication --------------- - -Request Authentication -~~~~~~~~~~~~~~~~~~~~~~ - -Every HTTP request made by a homeserver is authenticated using public key -digital signatures. The request method, target and body are signed by wrapping -them in a JSON object and signing it using the JSON signing algorithm. The -resulting signatures are added as an Authorization header with an auth scheme -of ``X-Matrix``. Note that the target field should include the full path -starting with ``/_matrix/...``, including the ``?`` and any query parameters if -present, but should not include the leading ``https:``, nor the destination -server's hostname. - -Step 1 sign JSON: - -.. code:: - - { - "method": "GET", - "uri": "/target", - "origin": "origin.hs.example.com", - "destination": "destination.hs.example.com", - "content": , - "signatures": { - "origin.hs.example.com": { - "ed25519:key1": "ABCDEF..." - } - } - } - -Step 2 add Authorization header: - -.. code:: - - GET /target HTTP/1.1 - Authorization: X-Matrix origin=origin.example.com,key="ed25519:key1",sig="ABCDEF..." - Content-Type: application/json - - - - -Example python code: - -.. code:: python - - def authorization_headers(origin_name, origin_signing_key, - destination_name, request_method, request_target, - content=None): - request_json = { - "method": request_method, - "uri": request_target, - "origin": origin_name, - "destination": destination_name, - } - - if content_json is not None: - request["content"] = content - - signed_json = sign_json(request_json, origin_name, origin_signing_key) - - authorization_headers = [] - - for key, sig in signed_json["signatures"][origin_name].items(): - authorization_headers.append(bytes( - "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % ( - origin_name, key, sig, - ) - )) - - return ("Authorization", authorization_headers) - -Response Authentication -~~~~~~~~~~~~~~~~~~~~~~~ - -Responses are authenticated by the TLS server certificate. A homeserver should -not send a request until it has authenticated the connected server to avoid -leaking messages to eavesdroppers. - -Client TLS Certificates -~~~~~~~~~~~~~~~~~~~~~~~ - -Requests are authenticated at the HTTP layer rather than at the TLS layer -because HTTP services like Matrix are often deployed behind load balancers that -handle the TLS and these load balancers make it difficult to check TLS client -certificates. - -A homeserver may provide a TLS client certificate and the receiving homeserver -may check that the client certificate matches the certificate of the origin -homeserver. - Public Room Directory --------------------- From 50fc1c4d16aa41b9c877c1d24fbd93c40c033648 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 9 Aug 2018 08:35:24 -0600 Subject: [PATCH 299/308] Move backfill/get events so that joins, leaves, and invites are together --- specification/server_server_api.rst | 76 ++++++++++++++--------------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index b045db72..472aca12 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -576,7 +576,44 @@ A *conflict* occurs between states where those states have different affected are said to be *conflicting* events. +Backfilling and retrieving missing events +----------------------------------------- +Once a homeserver has joined a room, it receives all the events emitted by +other homeservers in that room, and is thus aware of the entire history of the +room from that moment onwards. Since users in that room are able to request the +history by the ``/messages`` client API endpoint, it's possible that they might +step backwards far enough into history before the homeserver itself was a +member of that room. + +To cover this case, the federation API provides a server-to-server analog of +the ``/messages`` client API, allowing one homeserver to fetch history from +another. This is the ``/backfill`` API. + +To request more history, the requesting homeserver picks another homeserver +that it thinks may have more (most likely this should be a homeserver for +some of the existing users in the room at the earliest point in history it +has currently), and makes a ``/backfill`` request. + +Similar to backfilling a room's history, a server may not have all the events +in the graph. That server may use the ``/get_missing_events`` API to acquire +the events it is missing. + +.. TODO-spec + Specify (or remark that it is unspecified) how the server handles divergent + history. DFS? BFS? Anything weirder? + +{{backfill_ss_http_api}} + +Retrieving events +---------------- + +In some circumstances, a homeserver may be missing a particular event or information +about the room which cannot be easily determined from backfilling. These APIs provide +homeservers with the option of getting events and the state of the room at a given +point in the timeline. + +{{events_ss_http_api}} Joining Rooms @@ -664,45 +701,6 @@ participating in the room. here. What purpose does it serve expanding them out in full, when surely they'll appear in the state anyway? -Backfilling and retrieving missing events ------------------------------------------ - -Once a homeserver has joined a room, it receives all the events emitted by -other homeservers in that room, and is thus aware of the entire history of the -room from that moment onwards. Since users in that room are able to request the -history by the ``/messages`` client API endpoint, it's possible that they might -step backwards far enough into history before the homeserver itself was a -member of that room. - -To cover this case, the federation API provides a server-to-server analog of -the ``/messages`` client API, allowing one homeserver to fetch history from -another. This is the ``/backfill`` API. - -To request more history, the requesting homeserver picks another homeserver -that it thinks may have more (most likely this should be a homeserver for -some of the existing users in the room at the earliest point in history it -has currently), and makes a ``/backfill`` request. - -Similar to backfilling a room's history, a server may not have all the events -in the graph. That server may use the ``/get_missing_events`` API to acquire -the events it is missing. - -.. TODO-spec - Specify (or remark that it is unspecified) how the server handles divergent - history. DFS? BFS? Anything weirder? - -{{backfill_ss_http_api}} - -Retrieving events ----------------- - -In some circumstances, a homeserver may be missing a particular event or information -about the room which cannot be easily determined from backfilling. These APIs provide -homeservers with the option of getting events and the state of the room at a given -point in the timeline. - -{{events_ss_http_api}} - Inviting to a room ------------------ From 292d3345099a80d6de88035d6859400b1c10be47 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Fri, 10 Aug 2018 16:19:17 -0400 Subject: [PATCH 300/308] document new login identifier object (#1390) --- .../definitions/user_identifier.yaml | 24 ++++ api/client-server/login.yaml | 15 ++- .../client_server/newsfragments/1390.feature | 1 + specification/client_server_api.rst | 114 ++++++++++++++++-- 4 files changed, 143 insertions(+), 11 deletions(-) create mode 100644 api/client-server/definitions/user_identifier.yaml create mode 100644 changelogs/client_server/newsfragments/1390.feature diff --git a/api/client-server/definitions/user_identifier.yaml b/api/client-server/definitions/user_identifier.yaml new file mode 100644 index 00000000..ce65053d --- /dev/null +++ b/api/client-server/definitions/user_identifier.yaml @@ -0,0 +1,24 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +title: User identifier +description: |- + Identification information for a user +type: object +properties: + type: + type: string + description: The type of identification. See `Identifier types`_ for supported values and additional property descriptions. +required: + - type +additionalProperties: true diff --git a/api/client-server/login.yaml b/api/client-server/login.yaml index 289b17e7..43aae5df 100644 --- a/api/client-server/login.yaml +++ b/api/client-server/login.yaml @@ -1,4 +1,5 @@ # Copyright 2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -85,7 +86,10 @@ paths: type: object example: { "type": "m.login.password", - "user": "cheeky_monkey", + "identifier": { + "type": "m.id.user", + "user": "cheeky_monkey" + }, "password": "ilovebananas", "initial_device_display_name": "Jungle Phone" } @@ -94,15 +98,18 @@ paths: type: string enum: ["m.login.password", "m.login.token"] description: The login type being used. + identifier: + description: Identification information for the user. + "$ref": "definitions/user_identifier.yaml" user: type: string - description: The fully qualified user ID or just local part of the user ID, to log in. + description: The fully qualified user ID or just local part of the user ID, to log in. Deprecated in favour of ``identifier``. medium: type: string - description: When logging in using a third party identifier, the medium of the identifier. Must be 'email'. + description: When logging in using a third party identifier, the medium of the identifier. Must be 'email'. Deprecated in favour of ``identifier``. address: type: string - description: Third party identifier for the user. + description: Third party identifier for the user. Deprecated in favour of ``identifier``. password: type: string description: |- diff --git a/changelogs/client_server/newsfragments/1390.feature b/changelogs/client_server/newsfragments/1390.feature new file mode 100644 index 00000000..48a53b0a --- /dev/null +++ b/changelogs/client_server/newsfragments/1390.feature @@ -0,0 +1 @@ +Add new user identifier object for logging in \ No newline at end of file diff --git a/specification/client_server_api.rst b/specification/client_server_api.rst index 1bb989ff..58f68f25 100644 --- a/specification/client_server_api.rst +++ b/specification/client_server_api.rst @@ -461,7 +461,7 @@ Password-based :Type: ``m.login.password`` :Description: - The client submits a username and secret password, both sent in plain-text. + The client submits an identifier and secret password, both sent in plain-text. To use this authentication type, clients should submit an auth dict as follows: @@ -469,7 +469,26 @@ To use this authentication type, clients should submit an auth dict as follows: { "type": "m.login.password", - "user": "", + "identifier": { + ... + }, + "password": "", + "session": "" + } + +where the ``identifier`` property is a user identifier object, as described in +`Identifier types`_. + +For example, to authenticate using the user's Matrix ID, clients whould submit: + +.. code:: json + + { + "type": "m.login.password", + "identifier": { + "type": "m.id.user", + "user": "" + }, "password": "", "session": "" } @@ -482,8 +501,11 @@ follows: { "type": "m.login.password", - "medium": "", - "address": "", + "identifier": { + "type": "m.id.thirdparty", + "medium": "", + "address": "" + }, "password": "", "session": "" } @@ -714,6 +736,78 @@ handle unknown login types: } +Identifier types +++++++++++++++++ + +Some authentication mechanisms use a user identifier object to identify a +user. The user identifier object has a ``type`` field to indicate the type of +identifier being used, and depending on the type, has other fields giving the +information required to identify the user as described below. + +This specification defines the following identifier types: + - ``m.id.user`` + - ``m.id.thirdparty`` + - ``m.id.phone`` + +Matrix User ID +<<<<<<<<<<<<<< +:Type: + ``m.id.user`` +:Description: + The user is identified by their Matrix ID. + +A client can identify a user using their Matrix ID. This can either be the +fully qualified Matrix user ID, or just the localpart of the user ID. + +.. code:: json + + "identifier": { + "type": "m.id.user", + "user": "" + } + +Third-party ID +<<<<<<<<<<<<<< +:Type: + ``m.id.thirdparty`` +:Description: + The user is identified by a third-party identifer in canonicalised form. + +A client can identify a user using a 3pid associated with the user's account on +the homeserver, where the 3pid was previously associated using the +|/account/3pid|_ API. See the `3PID Types`_ Appendix for a list of Third-party +ID media. + +.. code:: json + + "identifier": { + "type": "m.id.thirdparty", + "medium": "", + "address": "" + } + +Phone number +<<<<<<<<<<<< +:Type: + ``m.id.phone`` +:Description: + The user is identified by a phone number. + +A client can identify a user using a phone number associated with the user's +account, where the phone number was previously associated using the +|/account/3pid|_ API. The phone number can be passed in as entered by the +user; the homeserver will be responsible for canonicalising it. If the client +wishes to canonicalise the phone number, then it can use the +``m.id.thirdparty`` identifier type with a ``medium`` of ``msisdn`` instead. + +.. code:: json + + "identifier": { + "type": "m.id.phone", + "country": "", + "phone": "" + } + Login ~~~~~ @@ -729,7 +823,10 @@ request as follows: { "type": "m.login.password", - "user": "", + "identifier": { + "type": "m.id.user", + "user": "" + }, "password": "" } @@ -741,8 +838,10 @@ explicitly, as follows: { "type": "m.login.password", - "medium": "", - "address": "", + "identifier": { + "medium": "", + "address": "" + }, "password": "" } @@ -1477,3 +1576,4 @@ have to wait in milliseconds before they can try again. .. _/user//account_data/: #put-matrix-client-%CLIENT_MAJOR_VERSION%-user-userid-account-data-type .. _`Unpadded Base64`: ../appendices.html#unpadded-base64 +.. _`3PID Types`: ../appendices.html#pid-types From cd4fed509c625451aa0f0e3bb33b8d41b53c1bd3 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Fri, 10 Aug 2018 16:21:06 -0400 Subject: [PATCH 301/308] convert IS endpoints to Swagger (#1427) --- api/identity/associations.yaml | 179 ++++++++++++++++++++++ api/identity/email_associations.yaml | 197 +++++++++++++++++++++++++ api/identity/invitation_signing.yaml | 90 +++++++++++ api/identity/store_invite.yaml | 114 ++++++++++++++ specification/identity_service_api.rst | 141 +----------------- 5 files changed, 587 insertions(+), 134 deletions(-) create mode 100644 api/identity/associations.yaml create mode 100644 api/identity/email_associations.yaml create mode 100644 api/identity/invitation_signing.yaml create mode 100644 api/identity/store_invite.yaml diff --git a/api/identity/associations.yaml b/api/identity/associations.yaml new file mode 100644 index 00000000..784bb5d6 --- /dev/null +++ b/api/identity/associations.yaml @@ -0,0 +1,179 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +swagger: '2.0' +info: + title: "Matrix Identity Service Establishing Associations API" + version: "1.0.0" +host: localhost:8090 +schemes: + - https + - http +basePath: /_matrix/identity/api/v1 +produces: + - application/json +paths: + "/3pid/getValidated3pid": + get: + summary: Check whether ownership of a 3pid was validated. + description: A client can check whether ownership of a 3pid was validated + operationId: getValidated3pid + parameters: + - in: query + type: string + name: sid + description: The Session ID generated by the ``requestToken`` call. + required: true + x-example: 1234 + - in: query + type: string + name: client_secret + description: The client secret passed to the ``requestToken`` call. + required: true + x-example: monkeys_are_GREAT + responses: + 200: + description: Validation information for the session. + examples: + application/json: { + "medium": "email", + "validated_at": 1457622739026, + "address": "louise@bobs.burgers" + } + schema: + type: object + properties: + medium: + type: string + description: The medium type of the 3pid. + address: + type: string + description: The address of the 3pid being looked up. + validated_at: + type: integer + description: Timestamp indicating the time that the 3pid was validated. + 400: + description: |- + The session has not been validated. + + If the session has not been validated, then ``errcode`` will be + ``M_SESSION_NOT_VALIDATED``. If the session has timed out, then + ``errcode`` will be ``M_SESSION_EXPIRED``. + examples: + application/json: { + "errcode": "M_SESSION_NOT_VALIDATED", + "error": "This validation session has not yet been completed" + } + 404: + description: The Session ID or client secret were not found + examples: + application/json: { + "errcode": "M_NO_VALID_SESSION", + "error": "No valid session was found matching that sid and client secret" + } + "/bind": + post: + summary: Publish an association between a session and a Matrix user ID. + description: |- + Publish an association between a session and a Matrix user ID. + + Future calls to ``/lookup`` for any of the session\'s 3pids will return + this association. + + Note: for backwards compatibility with older versions of this + specification, the parameters may also be specified as + ``application/x-form-www-urlencoded`` data. However, this usage is + deprecated. + operationId: bind + parameters: + - in: body + name: body + schema: + type: object + example: { + "sid": "1234", + "client_secret": "monkeys_are_GREAT", + "mxid": "@ears:matrix.org" + } + properties: + sid: + type: string + description: The Session ID generated by the ``requestToken`` call. + client_secret: + type: string + description: The client secret passed to the ``requestToken`` call. + mxid: + type: string + description: The Matrix user ID to associate with the 3pids. + required: ["sid", "client_secret", "mxid"] + responses: + 200: + description: The association was published. + examples: + application/json: { + "address": "louise@bobs.burgers", + "medium": "email", + "mxid": "@ears:matrix.org", + "not_before": 1428825849161, + "not_after": 4582425849161, + "ts": 1428825849161, + + "signatures": { + "matrix.org": { + "ed25519:0": "ENiU2YORYUJgE6WBMitU0mppbQjidDLanAusj8XS2nVRHPu+0t42OKA/r6zV6i2MzUbNQ3c3MiLScJuSsOiVDQ" + } + } + } + schema: + type: object + properties: + address: + type: string + description: The 3pid address of the user being looked up. + medium: + type: string + description: The medium type of the 3pid. + mxid: + type: string + description: The Matrix user ID associated with the 3pid. + not_before: + type: integer + description: A unix timestamp before which the association is not known to be valid. + not_after: + type: integer + description: A unix timestamp after which the association is not known to be valid. + ts: + type: integer + description: The unix timestamp at which the association was verified. + signatures: + type: object + description: The signatures of the verifying identity services which show that the association should be trusted, if you trust the verifying identity services. + 400: + description: |- + The association was not published. + + If the session has not been validated, then ``errcode`` will be + ``M_SESSION_NOT_VALIDATED``. If the session has timed out, then + ``errcode`` will be ``M_SESSION_EXPIRED``. + examples: + application/json: { + "errcode": "M_SESSION_NOT_VALIDATED", + "error": "This validation session has not yet been completed" + } + 404: + description: The Session ID or client secret were not found + examples: + application/json: { + "errcode": "M_NO_VALID_SESSION", + "error": "No valid session was found matching that sid and client secret" + } diff --git a/api/identity/email_associations.yaml b/api/identity/email_associations.yaml new file mode 100644 index 00000000..8431c9e8 --- /dev/null +++ b/api/identity/email_associations.yaml @@ -0,0 +1,197 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +swagger: '2.0' +info: + title: "Matrix Identity Service Email Associations API" + version: "1.0.0" +host: localhost:8090 +schemes: + - https + - http +basePath: /_matrix/identity/api/v1 +produces: + - application/json +paths: + "/validate/email/requestToken": + post: + summary: Request a token for validating an email address. + description: |- + Create a session for validating an email address. + + The identity service will send an email containing a token. If that + token is presented to the identity service in the future, it indicates + that that user was able to read the email for that email address, and + so we validate ownership of the email address. + + Note that Home Servers offer APIs that proxy this API, adding + additional behaviour on top, for example, + ``/register/email/requestToken`` is designed specifically for use when + registering an account and therefore will inform the user if the email + address given is already registered on the server. + + Note: for backwards compatibility with older versions of this + specification, the parameters may also be specified as + ``application/x-form-www-urlencoded`` data. However, this usage is + deprecated. + operationId: emailRequestToken + parameters: + - in: body + name: body + schema: + type: object + example: { + "client_secret": "monkeys_are_GREAT", + "email": "foo@example.com", + "send_attempt": 1 + } + properties: + client_secret: + type: string + description: A unique string used to identify the validation attempt + email: + type: string + description: The email address to validate. + send_attempt: + type: integer + description: |- + Optional. If specified, the server will only send an email if + the ``send_attempt`` is a number greater than the most recent + one which it has seen (or if it has never seen one), scoped + to that ``email`` + ``client_secret`` pair. This is to avoid + repeatedly sending the same email in the case of request + retries between the POSTing user and the identity + service. The client should increment this value if they + desire a new email (e.g. a reminder) to be sent. + next_link: + type: string + description: |- + Optional. When the validation is completed, the identity + service will redirect the user to this URL. + required: ["client_secret", "email"] + responses: + 200: + description: + Session created. + examples: + application/json: { + "sid": "1234" + } + schema: + type: object + properties: + sid: + type: string + description: The session ID. + 400: + description: | + An error ocurred. Some possible errors are: + + - ``M_INVALID_EMAIL``: The email address provided was invalid. + - ``M_EMAIL_SEND_ERROR``: The validation email could not be sent. + "/validate/email/submitToken": + post: + summary: Validate ownership of an email address. + description: |- + Validate ownership of an email address. + + If the three parameters are consistent with a set generated by a + ``requestToken`` call, ownership of the email address is considered to + have been validated. This does not publish any information publicly, or + associate the email address with any Matrix user ID. Specifically, + calls to ``/lookup`` will not show a binding. + + Note: for backwards compatibility with older versions of this + specification, the parameters may also be specified as + ``application/x-form-www-urlencoded`` data. However, this usage is + deprecated. + operationId: emailSubmitTokenPost + parameters: + - in: body + name: body + schema: + type: object + example: { + "sid": "1234", + "client_secret": "monkeys_are_GREAT", + "token": "atoken" + } + properties: + sid: + type: string + description: The session ID, generated by the ``requestToken`` call. + client_secret: + type: string + description: The client secret that was supplied to the ``requestToken`` call. + token: + type: string + description: The token generated by the ``requestToken`` call and emailed to the user. + required: ["sid", "client_secret", "token"] + responses: + 200: + description: + The success of the validation. + examples: + application/json: { + "success": true + } + schema: + type: object + properties: + success: + type: boolean + description: Whether the validation was successful or not. + get: + summary: Validate ownership of an email address. + description: |- + Validate ownership of an email address. + + If the three parameters are consistent with a set generated by a + ``requestToken`` call, ownership of the email address is considered to + have been validated. This does not publish any information publicly, or + associate the email address with any Matrix user ID. Specifically, + calls to ``/lookup`` will not show a binding. + + Note that, in contrast with the POST version, this endpoint will be + used by end-users, and so the response should be human-readable. + operationId: emailSubmitTokenGet + parameters: + - in: query + type: string + name: sid + required: true + description: The session ID, generated by the ``requestToken`` call. + x-example: 1234 + - in: query + type: string + name: client_secret + required: true + description: The client secret that was supplied to the ``requestToken`` call. + x-example: monkeys_are_GREAT + - in: query + type: string + name: token + required: true + description: The token generated by the ``requestToken`` call and emailed to the user. + x-example: atoken + responses: + "200": + description: Email address is validated. + "3xx": + description: |- + Email address is validated, and the ``next_link`` parameter was + provided to the ``requestToken`` call. The user must be redirected + to the URL provided by the ``next_link`` parameter. + "4xx": + description: + Validation failed. diff --git a/api/identity/invitation_signing.yaml b/api/identity/invitation_signing.yaml new file mode 100644 index 00000000..982dbff7 --- /dev/null +++ b/api/identity/invitation_signing.yaml @@ -0,0 +1,90 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +swagger: '2.0' +info: + title: "Matrix Identity Service Ephemeral Invitation Signing API" + version: "1.0.0" +host: localhost:8090 +schemes: + - https + - http +basePath: /_matrix/identity/api/v1 +produces: + - application/json +paths: + "/sign-ed25519": + post: + summary: Sign invitation details + description: |- + Sign invitation details. + + The identity server will look up ``token`` which was stored in a call + to ``store-invite``, and fetch the sender of the invite. + operationId: blindlySignStuff + parameters: + - in: body + name: body + schema: + type: object + example: { + "mxid": "@foo:bar.com", + "token": "sometoken", + "private_key": "base64encodedkey" + } + properties: + mxid: + type: string + description: The Matrix user ID of the user accepting the invitation. + token: + type: string + description: Token from the call to ``store-invite`` + private_key: + type: string + description: The private key, encoded as `Unpadded base64`_. + required: ["mxid", "token", "private_key"] + responses: + 200: + description: The signedjson of the mxid, sender, and token. + schema: + type: object + properties: + mxid: + type: string + description: The Matrix user ID of the user accepting the invitation. + sender: + type: string + description: The Matrix user ID of the user who sent the invitation. + signatures: + type: object + description: The signature of the mxid, sender, and token. + token: + type: string + description: The token for the invitation. + examples: + application/json: { + "mxid": "@foo:bar.com", + "sender": "@baz:bar.com", + "signatures": { + "my.id.server": { + "ed25519:0": "def987" + } + }, + "token": "abc123" + } + 404: + description: Token was not found. + example: { + "errcode": "M_UNRECOGNIZED", + "error": "Didn't recognize token" + } diff --git a/api/identity/store_invite.yaml b/api/identity/store_invite.yaml new file mode 100644 index 00000000..6b847b5b --- /dev/null +++ b/api/identity/store_invite.yaml @@ -0,0 +1,114 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +swagger: '2.0' +info: + title: "Matrix Identity Service Store Invitations API" + version: "1.0.0" +host: localhost:8090 +schemes: + - https + - http +basePath: /_matrix/identity/api/v1 +produces: + - application/json +paths: + "/store-invite": + post: + summary: Store pending invitations to a user\'s 3pid. + description: |- + Store pending invitations to a user\'s 3pid. + + In addition to the request parameters specified below, an arbitrary + number of other parameters may also be specified. These may be used in + the invite message generation described below. + + The service will generate a random token and an ephemeral key used for + accepting the invite. + + The service also generates a ``display_name`` for the inviter, which is + a redacted version of ``address`` which does not leak the full contents + of the ``address``. + + The service records persistently all of the above information. + + It also generates an email containing all of this data, sent to the + ``address`` parameter, notifying them of the invitation. + + Also, the generated ephemeral public key will be listed as valid on + requests to ``/_matrix/identity/api/v1/pubkey/ephemeral/isvalid``. + operationId: storeInvite + parameters: + - in: body + name: body + schema: + type: object + example: { + "medium": "email", + "address": "foo@bar.baz", + "room_id": "!something:example.tld", + "sender": "@bob:example.com" + } + properties: + medium: + type: string + description: The literal string ``email``. + address: + type: string + description: The email address of the invited user. + room_id: + type: string + description: The Matrix room ID to which the user is invited + sender: + type: string + description: The Matrix user ID of the inviting user + required: ["medium", "address", "room_id", "sender"] + responses: + 200: + description: The invitation was stored. + schema: + type: object + properties: + token: + type: string + description: The generated token. + public_keys: + type: array + description: A list of [server\'s long-term public key, generated ephemeral public key]. + items: + type: string + display_name: + type: string + description: The generated (redacted) display_name. + example: + application/json: { + "token": "sometoken", + "public_keys": [ + "serverpublickey", + "ephemeralpublickey" + ], + "display_name": "f...@b..." + } + 400: + description: | + An error has occured. + + If the 3pid is already bound to a Matrix user ID, the error code + will be ``M_THREEPID_IN_USE``. If the medium is unsupported, the + error code will be ``M_UNRECOGNIZED``. + examples: + application/json: { + "errcode": "M_THREEPID_IN_USE", + "error": "Binding already known", + "mxid": mxid + } diff --git a/specification/identity_service_api.rst b/specification/identity_service_api.rst index 89fcc3e6..cb079593 100644 --- a/specification/identity_service_api.rst +++ b/specification/identity_service_api.rst @@ -114,100 +114,15 @@ session, within a 24 hour period since its most recent modification. Any attempts to perform these actions after the expiry will be rejected, and a new session should be created and used instead. -Creating a session +Email associations ~~~~~~~~~~~~~~~~~~ -A client makes a call to:: +{{email_associations_is_http_api}} - POST https://my.id.server:8090/_matrix/identity/api/v1/validate/email/requestToken +General +~~~~~~~ - client_secret=monkeys_are_GREAT& - email=foo@bar.com& - send_attempt=1 - -It may also optionally specify next_link. If next_link is specified, when the -validation is completed, the identity service will redirect the user to that -URL. - -This will create a new "session" on the identity service, identified by an -``sid``. - -The identity service will send an email containing a token. If that token is -presented to the identity service in the future, it indicates that that user was -able to read the email for that email address, and so we validate ownership of -the email address. - -We return the ``sid`` generated for this session to the caller, in a JSON object -containing the ``sid`` key. - -If a send_attempt is specified, the server will only send an email if the -send_attempt is a number greater than the most recent one which it has seen (or -if it has never seen one), scoped to that email address + client_secret pair. -This is to avoid repeatedly sending the same email in the case of request -retries between the POSTing user and the identity service. The client should -increment this value if they desire a new email (e.g. a reminder) to be sent. - -Note that Home Servers offer APIs that proxy this API, adding additional -behaviour on top, for example, ``/register/email/requestToken`` is designed -specifically for use when registering an account and therefore will inform -the user if the email address given is already registered on the server. - -Validating ownership of an email -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A user may make either a ``GET`` or a ``POST`` request to -``/_matrix/identity/api/v1/validate/email/submitToken`` with the following -parameters (either as query parameters or URL-encoded POST parameters): -- ``sid`` the sid for the session, generated by the ``requestToken`` call. -- ``client_secret`` the client secret which was supplied to the ``requestToken`` call. -- ``token`` the token generated by the ``requestToken`` call, and emailed to the user. - -If these three values are consistent with a set generated by a ``requestToken`` -call, ownership of the email address is considered to have been validated. This -does not publish any information publicly, or associate the email address with -any Matrix user ID. Specifically, calls to ``/lookup`` will not show a binding. - -Otherwise, an error will be returned. - -Checking non-published 3pid ownership -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A client can check whether ownership of a 3pid was validated by making an -HTTP GET request to ``/_matrix/identity/api/v1/3pid/getValidated3pid``, passing -the ``sid`` and ``client_secret`` as query parameters from the ``requestToken`` -call. - -It will return something of either the form:: - - {"medium": "email", "validated_at": 1457622739026, "address": "foo@bar.com"} - -or:: - - {"errcode": "M_SESSION_NOT_VALIDATED", "error": "This validation session has not yet been completed"} - -If the ``sid`` and ``client_secret`` were not recognised, or were not correct, -an error will be returned. - -Publishing a validated association -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -An association between a session and a Matrix user ID can be published by making -a URL-encoded HTTP POST request to ``/_matrix/identity/api/v1/3pid/bind`` with -the following parameters:: - - sid=sid& - client_secret=monkeys_are_GREAT& - mxid=@foo:bar.com - -If the session is still valid, this will publish an association between the -3pids validated on that session and the passed Matrix user ID. Future calls -to ``/lookup`` for any of the session's 3pids will return this association. - -If the 3pid has not yet been validated, the HTTP request will be rejected, and -the association will not be established. - -If the ``sid`` and ``client_secret`` were not recognised, or were not correct, -an error will be returned. +{{associations_is_http_api}} Invitation Storage ------------------ @@ -216,32 +131,6 @@ An identity service can store pending invitations to a user's 3pid, which will be retrieved and can be either notified on or look up when the 3pid is associated with a Matrix user ID. -If one makes a ``POST`` request to ``/_matrix/identity/api/v1/store-invite`` with the following URL-encoded POST parameters: - -- ``medium`` (string, required): The literal string ``email``. -- ``address`` (string, required): The email address of the invited user. -- ``room_id`` (string, required): The Matrix room ID to which the user is invited. -- ``sender`` (string, required): The matrix user ID of the inviting user. - -An arbitrary number of other parameters may also be specified. These may be used in the email generation described below. - -The service will look up whether the 3pid is bound to a Matrix user ID. If it is, the request will be rejected with a 400 status code. - -If the medium is something other than the literal string ``email``, the request will be rejected with a 400 status code. - -Otherwise, the service will then generate a random string called ``token``, and an ephemeral public key. - -The service also generates a ``display_name`` for the inviter, which is a redacted version of ``address`` which does not leak the full contents of the ``address``. - -The service records persistently all of the above information. - -It also generates an email containing all of this data, sent to the ``address`` parameter, notifying them of the invitation. - -The response body is then populated as the JSON-encoded dictionary containing the following fields: -- ``token`` (string): The generated token. -- ``public_keys`` ([string]): A list of [server's long-term public key, generated ephemeral public key]. -- ``display_name`` (string): The generated (redacted) display_name. - At a later point, if the owner of that particular 3pid binds it with a Matrix user ID, the identity server will attempt to make an HTTP POST to the Matrix user's homeserver which looks roughly as below:: POST https://bar.com:8448/_matrix/federation/v1/3pid/onbind @@ -273,7 +162,7 @@ At a later point, if the owner of that particular 3pid binds it with a Matrix us Where the signature is produced using a long-term private key. -Also, the generated ephemeral public key will be listed as valid on requests to ``/_matrix/identity/api/v1/pubkey/ephemeral/isvalid``. +{{store_invite_is_http_api}} Ephemeral invitation signing ---------------------------- @@ -281,23 +170,7 @@ Ephemeral invitation signing To aid clients who may not be able to perform crypto themselves, the identity service offers some crypto functionality to help in accepting invitations. This is less secure than the client doing it itself, but may be useful where this isn't possible. -The identity service will happily sign invitation details with a request-specified ed25519 private key for you, if you want it to. It takes URL-encoded POST parameters: -- mxid (string, required) -- token (string, required) -- private_key (string, required): The private key, encoded as `Unpadded base64`_. - -It will look up ``token`` which was stored in a call to ``store-invite``, and fetch the sender of the invite. It will then respond with JSON which looks something like:: - - { - "mxid": "@foo:bar.com", - "sender": "@baz:bar.com", - "signatures" { - "my.id.server": { - "ed25519:0": "def987" - } - }, - "token": "abc123" - } +{{invitation_signing_is_http_api}} .. _`Unpadded Base64`: ../appendices.html#unpadded-base64 .. _`3PID Types`: ../appendices.html#pid-types From 88d97b06ae5cd7816f4297e67e4701e4c2e8af82 Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Sat, 28 Jul 2018 22:54:59 +0900 Subject: [PATCH 302/308] Support oneOf --- scripts/templating/matrix_templates/units.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 88f7b86c..5930b0d0 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -286,7 +286,9 @@ def get_json_schema_object_fields(obj, enforce_title=False): def process_data_type(prop, required=False, enforce_title=True): prop = inherit_parents(prop) - prop_type = prop['type'] + prop_type = prop.get('oneOf', prop.get('type', [])) + assert prop_type + tables = [] enum_desc = None is_object = False @@ -317,6 +319,15 @@ def process_data_type(prop, required=False, enforce_title=True): tables = nested["tables"] enum_desc = nested["enum_desc"] + elif isinstance(prop_type, list): + prop_title = [] + for t in prop_type: + if isinstance(t, dict): + nested = process_data_type(t) + tables.extend(nested['tables']) + prop_title.append(nested['title']) + else: + prop_title.append(t) else: prop_title = prop_type From 9be78f3aa7f6b7577aef6891c5d47760203107da Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Sun, 29 Jul 2018 14:58:37 +0900 Subject: [PATCH 303/308] Better support enums when they come among oneOf types --- scripts/templating/matrix_templates/units.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/templating/matrix_templates/units.py b/scripts/templating/matrix_templates/units.py index 5930b0d0..90a87cd4 100644 --- a/scripts/templating/matrix_templates/units.py +++ b/scripts/templating/matrix_templates/units.py @@ -326,6 +326,10 @@ def process_data_type(prop, required=False, enforce_title=True): nested = process_data_type(t) tables.extend(nested['tables']) prop_title.append(nested['title']) + # Assuming there's at most one enum among type options + enum_desc = nested['enum_desc'] + if enum_desc: + enum_desc = "%s if the type is enum" % enum_desc else: prop_title.append(t) else: From b41eb26124de0f5c6b54fba4a9cb3939fb10adfe Mon Sep 17 00:00:00 2001 From: Kitsune Ral Date: Sun, 12 Aug 2018 16:52:07 +0900 Subject: [PATCH 304/308] Document OpenAPI extension Signed-off-by: Alexey Rusakov --- api/openapi_extensions.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/api/openapi_extensions.md b/api/openapi_extensions.md index 9f4745fd..339452ba 100644 --- a/api/openapi_extensions.md +++ b/api/openapi_extensions.md @@ -21,3 +21,25 @@ Example: name: fields... type: string ``` + +## Using oneOf to provide type alternatives + + + +`oneOf` (available in JSON Schema and Swagger/OpenAPI v3 but not in v2) +is used in cases when a simpler type specification as a list of types +doesn't work, as in the following example: +``` + properties: + old: # compliant with old Swagger + type: + - string + - object # Cannot specify a schema here + new: # uses oneOf extension + oneOf: + - type: string + - type: object + title: CustomSchemaForTheWin + properties: + ... +``` From 6ad71f785e833f4e140647bd3a76f3af748b38fc Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 14 Aug 2018 12:48:59 -0600 Subject: [PATCH 305/308] Improve documentation for pushers and push gateways This fixes a number of formatting issues alongside a few documentation problems: * The push gateway can actually expect less parameters than previously advertised. This is for user privacy. * Introduction of the `m.email` pusher for email-capable homeservers. * Fields not being flagged as required on some endpoints. * Document the `event_id_only` format Note: this does not attempt to document push rules, just pushers. Fixes https://github.com/matrix-org/matrix-doc/issues/1374 Fixes https://github.com/matrix-org/matrix-doc/issues/1087 --- api/client-server/pusher.yaml | 106 ++++++++++++++++++---------- api/push-gateway/push_notifier.yaml | 90 +++++++++++------------ specification/modules/push.rst | 3 + 3 files changed, 116 insertions(+), 83 deletions(-) diff --git a/api/client-server/pusher.yaml b/api/client-server/pusher.yaml index 938014c6..0e9cf641 100644 --- a/api/client-server/pusher.yaml +++ b/api/client-server/pusher.yaml @@ -1,4 +1,5 @@ # Copyright 2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -31,30 +32,30 @@ paths: get: summary: Gets the current pushers for the authenticated user description: |- - Gets all currently active pushers for the authenticated user + Gets all currently active pushers for the authenticated user. operationId: getPushers security: - accessToken: [] responses: 200: - description: The pushers for this user + description: The pushers for this user. examples: application/json: { - "pushers": [ - { - "pushkey": "Xp/MzCt8/9DcSNE9cuiaoT5Ac55job3TdLSSmtmYl4A=", - "kind": "http", - "app_id": "face.mcapp.appy.prod", - "app_display_name": "Appy McAppface", - "device_display_name": "Alice's Phone", - "profile_tag": "xyz", - "lang": "en-US", - "data": { - "url": "https://example.com/_matrix/push/v1/notify" - } + "pushers": [ + { + "pushkey": "Xp/MzCt8/9DcSNE9cuiaoT5Ac55job3TdLSSmtmYl4A=", + "kind": "http", + "app_id": "face.mcapp.appy.prod", + "app_display_name": "Appy McAppface", + "device_display_name": "Alice's Phone", + "profile_tag": "xyz", + "lang": "en-US", + "data": { + "url": "https://example.com/_matrix/push/v1/notify" } - ] - } + } + ] + } schema: type: object properties: @@ -70,7 +71,7 @@ paths: pushkey: type: string description: |- - This is a unique identifier for this pusher. See `/set` for + This is a unique identifier for this pusher. See ``/set`` for more detail. Max length, 512 bytes. kind: @@ -115,6 +116,19 @@ paths: description: |- Required if ``kind`` is ``http``. The URL to use to send notifications to. + format: + type: string + description: |- + The format to use when sending notifications to the Push + Gateway. + required: + - pushkey + - app_id + - kind + - app_display_name + - device_display_name + - lang + - data tags: - Push notifications "/pushers/set": @@ -130,23 +144,24 @@ paths: parameters: - in: body name: pusher - description: The pusher information + description: The pusher information. required: true schema: type: object example: { - "lang": "en", - "kind": "http", - "app_display_name": "Mat Rix", - "device_display_name": "iPhone 9", - "profile_tag": "xxyyzz", - "app_id": "com.example.app.ios", - "pushkey": "APA91bHPRgkF3JUikC4ENAHEeMrd41Zxv3hVZjC9KtT8OvPVGJ-hQMRKRrZuJAEcl7B338qju59zJMjw2DELjzEvxwYv7hH5Ynpc1ODQ0aT4U4OFEeco8ohsN5PjL1iC2dNtk2BAokeMCg2ZXKqpc8FXKmhX94kIxQ", - "data": { - "url": "https://push-gateway.location.here" - }, - "append": false - } + "lang": "en", + "kind": "http", + "app_display_name": "Mat Rix", + "device_display_name": "iPhone 9", + "profile_tag": "xxyyzz", + "app_id": "com.example.app.ios", + "pushkey": "APA91bHPRgkF3JUikC4ENAHEeMrd41Zxv3hVZjC9KtT8OvPVGJ-hQMRKRrZuJAEcl7B338qju59zJMjw2DELjzEvxwYv7hH5Ynpc1ODQ0aT4U4OFEeco8ohsN5PjL1iC2dNtk2BAokeMCg2ZXKqpc8FXKmhX94kIxQ", + "data": { + "url": "https://push-gateway.location.here/_matrix/push/v1/notify", + "format": "event_id_only" + }, + "append": false + } properties: pushkey: type: string @@ -157,11 +172,15 @@ paths: for APNS or the Registration ID for GCM. If your notification client has no such concept, use any unique identifier. Max length, 512 bytes. + + If the ``kind`` is ``"email"``, this is the email address to + send notifications to. kind: type: string description: |- The kind of pusher to configure. ``"http"`` makes a pusher that - sends HTTP pokes. ``null`` deletes the pusher. + sends HTTP pokes. ``"email"`` makes a pusher that emails the + user with unread notifications. ``null`` deletes the pusher. app_id: type: string description: |- @@ -169,6 +188,8 @@ paths: It is recommended that this end with the platform, such that different platform versions get different app identifiers. Max length, 64 chars. + + If the ``kind`` is ``"email"``, this is ``"m.email"``. app_display_name: type: string description: |- @@ -188,7 +209,7 @@ paths: type: string description: |- The preferred language for receiving notifications (e.g. 'en' - or 'en-US') + or 'en-US'). data: type: object description: |- @@ -202,6 +223,15 @@ paths: description: |- Required if ``kind`` is ``http``. The URL to use to send notifications to. + format: + type: string + description: |- + The format to send notifications in to Push Gateways. If + unset or unrecognized, the homeserver should populate as + many fields as it can in the push notification. If set to + ``"event_id_only"``, the homeserver should only send the + absolute minimum amount of information possible to the push + gateway by not including optional fields. append: type: boolean description: |- @@ -216,17 +246,17 @@ paths: 200: description: The pusher was set. examples: - application/json: { - } + application/json: {} schema: - type: object # empty json object + type: object + description: An empty object. 400: description: One or more of the pusher values were invalid. examples: application/json: { - "error": "Missing parameters: lang, data", - "errcode": "M_MISSING_PARAM" - } + "error": "Missing parameters: lang, data", + "errcode": "M_MISSING_PARAM" + } schema: "$ref": "definitions/errors/error.yaml" 429: diff --git a/api/push-gateway/push_notifier.yaml b/api/push-gateway/push_notifier.yaml index 9b6e78d3..9f57b00a 100644 --- a/api/push-gateway/push_notifier.yaml +++ b/api/push-gateway/push_notifier.yaml @@ -1,4 +1,5 @@ # Copyright 2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,7 +20,7 @@ host: localhost:8008 schemes: - https - http -basePath: /_matrix/push/%CLIENT_MAJOR_VERSION% +basePath: /_matrix/push/v1 consumes: - application/json produces: @@ -38,14 +39,14 @@ paths: Notifications about a particular event will normally cause the user to be alerted in some way. It is therefore necessary to perform duplicate - suppression for such notifications using the `event_id` field to avoid + suppression for such notifications using the ``event_id`` field to avoid retries of this HTTP API causing duplicate alerts. The operation of updating counts of unread notifications should be idempotent and therefore do not require duplicate suppression. - Notifications are sent to the URL configured when the pusher is - created. This means that the HTTP path may be different depending on the - push gateway. + Notifications are sent to the URL configured when the pusher is created. + This means that the HTTP path may be different depending on the push + gateway. operationId: notify parameters: - in: body @@ -55,36 +56,36 @@ paths: schema: type: object example: { - "notification": { - "id": "$3957tyerfgewrf384", - "room_id": "!slw48wfj34rtnrf:example.com", - "type": "m.room.message", - "sender": "@exampleuser:matrix.org", - "sender_display_name": "Major Tom", - "room_name": "Mission Control", - "room_alias": "#exampleroom:matrix.org", - "prio": "high", - "content": { - "msgtype": "m.text", - "body": "I'm floating in a most peculiar way." - }, - "counts": { - "unread" : 2, - "missed_calls": 1 - }, - "devices": [ - { - "app_id": "org.matrix.matrixConsole.ios", - "pushkey": "V2h5IG9uIGVhcnRoIGRpZCB5b3UgZGVjb2RlIHRoaXM/", - "pushkey_ts": 12345678, - "data" : {}, - "tweaks": { - "sound": "bing" - } + "notification": { + "id": "$3957tyerfgewrf384", + "room_id": "!slw48wfj34rtnrf:example.com", + "type": "m.room.message", + "sender": "@exampleuser:matrix.org", + "sender_display_name": "Major Tom", + "room_name": "Mission Control", + "room_alias": "#exampleroom:matrix.org", + "prio": "high", + "content": { + "msgtype": "m.text", + "body": "I'm floating in a most peculiar way." + }, + "counts": { + "unread" : 2, + "missed_calls": 1 + }, + "devices": [ + { + "app_id": "org.matrix.matrixConsole.ios", + "pushkey": "V2h5IG9uIGVhcnRoIGRpZCB5b3UgZGVjb2RlIHRoaXM/", + "pushkey_ts": 12345678, + "data" : {}, + "tweaks": { + "sound": "bing" } - ] - } + } + ] } + } required: ["notification"] properties: notification: @@ -111,14 +112,10 @@ paths: type: string description: |- The type of the event as in the event's ``type`` field. - Required if the notification relates to a specific - Matrix event. sender: type: string description: |- The sender of the event as in the corresponding event field. - Required if the notification relates to a specific - Matrix event. sender_display_name: type: string description: |- @@ -149,14 +146,15 @@ paths: title: EventContent description: |- The ``content`` field from the event, if present. If the - event had no content field, this field is omitted. + event had no content field or the pusher wishes to not include + it, this field is omitted. counts: type: object title: Counts description: |- This is a dictionary of the current number of unacknowledged communications for the recipient user. Counts whose value is - zero are omitted. + zero should be omitted. properties: unread: type: integer @@ -180,10 +178,10 @@ paths: app_id: type: string description: |- - The app_id given when the pusher was created. + The ``app_id`` given when the pusher was created. pushkey: type: string - description: The pushkey given when the pusher was created. + description: The ``pushkey`` given when the pusher was created. pushkey_ts: type: integer description: |- @@ -202,13 +200,14 @@ paths: description: |- A dictionary of customisations made to the way this notification is to be presented. These are added by push rules. + required: ['app_id', 'pushkey'] responses: 200: description: A list of rejected push keys. examples: application/json: { - "rejected": [ "V2h5IG9uIGVhcnRoIGRpZCB5b3UgZGVjb2RlIHRoaXM/" ] - } + "rejected": [ "V2h5IG9uIGVhcnRoIGRpZCB5b3UgZGVjb2RlIHRoaXM/" ] + } schema: type: object # empty json object properties: @@ -222,7 +221,8 @@ paths: pushkeys and remove the associated pushers. It may not necessarily be the notification in the request that failed: it could be that a previous notification to the same pushkey - failed. + failed. May be empty. items: type: string - description: A pushkey + description: A pushkey that has been rejected. + required: ['rejected'] diff --git a/specification/modules/push.rst b/specification/modules/push.rst index 9bb65b96..e6a7acf5 100644 --- a/specification/modules/push.rst +++ b/specification/modules/push.rst @@ -65,6 +65,9 @@ APNS or Google's GCM. This happens as follows: notifications. 5. The Push Provider sends the notification to the device. +Homeservers may optionally support email notifications or other push kinds +of push, identified by the ``kind`` field of the pusher configuration. + Definitions for terms used in this section are below: Push Provider From de961fb7d6c37bf9404e07a91c7e60db64465425 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 15 Aug 2018 11:59:58 -0600 Subject: [PATCH 306/308] Improve wording on push gateways --- api/client-server/pusher.yaml | 11 +++++------ api/push-gateway/push_notifier.yaml | 6 +++--- specification/modules/push.rst | 5 ++--- specification/push_gateway.rst | 5 +++++ 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/api/client-server/pusher.yaml b/api/client-server/pusher.yaml index 0e9cf641..34050d3f 100644 --- a/api/client-server/pusher.yaml +++ b/api/client-server/pusher.yaml @@ -226,12 +226,11 @@ paths: format: type: string description: |- - The format to send notifications in to Push Gateways. If - unset or unrecognized, the homeserver should populate as - many fields as it can in the push notification. If set to - ``"event_id_only"``, the homeserver should only send the - absolute minimum amount of information possible to the push - gateway by not including optional fields. + The format to send notifications in to Push Gateways if the + ``kind`` is ``http``. The details about what fields the + homeserver should send to the push gateway are defined in the + `Push Gateway Specification`_. Currently the only format + available is 'event_id_only'. append: type: boolean description: |- diff --git a/api/push-gateway/push_notifier.yaml b/api/push-gateway/push_notifier.yaml index 9f57b00a..4a6cb8f7 100644 --- a/api/push-gateway/push_notifier.yaml +++ b/api/push-gateway/push_notifier.yaml @@ -145,9 +145,9 @@ paths: type: object title: EventContent description: |- - The ``content`` field from the event, if present. If the - event had no content field or the pusher wishes to not include - it, this field is omitted. + The ``content`` field from the event, if present. The pusher + may omit this if the event had no content or for any other + reason. counts: type: object title: Counts diff --git a/specification/modules/push.rst b/specification/modules/push.rst index e6a7acf5..e9ee8c90 100644 --- a/specification/modules/push.rst +++ b/specification/modules/push.rst @@ -65,9 +65,6 @@ APNS or Google's GCM. This happens as follows: notifications. 5. The Push Provider sends the notification to the device. -Homeservers may optionally support email notifications or other push kinds -of push, identified by the ``kind`` field of the pusher configuration. - Definitions for terms used in this section are below: Push Provider @@ -625,3 +622,5 @@ shouldn't be sent in the push itself where possible. Instead, Push Gateways should send a "sync" command to instruct the client to get new events from the homeserver directly. + +.. _`Push Gateway Specification`: ../push_gateway/unstable.html diff --git a/specification/push_gateway.rst b/specification/push_gateway.rst index 29a41bf7..e4a9d6ea 100644 --- a/specification/push_gateway.rst +++ b/specification/push_gateway.rst @@ -67,4 +67,9 @@ This describes the format used by "HTTP" pushers to send notifications of events to Push Gateways. If the endpoint returns an HTTP error code, the homeserver SHOULD retry for a reasonable amount of time using exponential backoff. +When pushing notifications for events, the hoemserver is expected to include all of +the event-related fields in the ``/notify`` request. When the homeserver is performing +a push where the ``format`` is ``"event_id_only"``, only the ``event_id``, ``room_id``, +``counts``, and ``devices`` are required to be populated. + {{push_notifier_push_http_api}} From ae02e8834d99099d6f5933e52840e042836e3867 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 15 Aug 2018 16:25:04 -0600 Subject: [PATCH 307/308] changelog --- changelogs/client_server/newsfragments/1506.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1506.feature diff --git a/changelogs/client_server/newsfragments/1506.feature b/changelogs/client_server/newsfragments/1506.feature new file mode 100644 index 00000000..62ad1260 --- /dev/null +++ b/changelogs/client_server/newsfragments/1506.feature @@ -0,0 +1 @@ +Document and improve client interaction with pushers. From 2eab07ade4ffe0a619e315436ceb947124ec6566 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 16 Aug 2018 11:45:20 -0600 Subject: [PATCH 308/308] Fix header in server-server API --- specification/server_server_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/server_server_api.rst b/specification/server_server_api.rst index 5f791a70..b5676b78 100644 --- a/specification/server_server_api.rst +++ b/specification/server_server_api.rst @@ -606,7 +606,7 @@ the events it is missing. {{backfill_ss_http_api}} Retrieving events ----------------- +----------------- In some circumstances, a homeserver may be missing a particular event or information about the room which cannot be easily determined from backfilling. These APIs provide