Merge branch 'main' into type-not-title-for-additional-pattern

Signed-off-by: Kévin Commaille <zecakeh@tedomum.fr>

.github/_typos.toml

@@ -10,3 +10,4 @@ au1ba7o = "au1ba7o"
 [default.extend-words]
 Appy = "Appy"
 fo = "fo"
+Iy = "Iy"

.github/workflows/main.yml

@@ -20,9 +20,9 @@ jobs:
       - name: "📥 Source checkout"
         uses: actions/checkout@v4
       - name: "➕ Setup Node"
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: '18'
+          node-version: '20'
       - name: "🔎 Run validator"
         run: |
           npx @redocly/cli@latest lint data/api/*/*.yaml
@@ -34,7 +34,7 @@ jobs:
       - name: "📥 Source checkout"
         uses: actions/checkout@v4
       - name: "➕ Setup Python"
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.9'
           cache: 'pip'
@@ -51,9 +51,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: "📥 Source checkout"
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: "➕ Setup Python"
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.9'
           cache: 'pip'
@@ -70,9 +70,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: "📥 Source checkout"
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: "➕ Setup Python"
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.9'
           cache: 'pip'
@@ -99,11 +99,11 @@ jobs:
         # the asterisk matching behaviour, not the literal string.
         run: |
           if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
-              echo ::set-output name=baseURL::/
+              echo "baseURL=/" >> "$GITHUB_OUTPUT"
           elif [[ "${GITHUB_REF}" == refs/tags/* ]]; then
-              echo ::set-output name=baseURL::"/${GITHUB_REF/refs\/tags\//}"
+              echo "baseURL=/${GITHUB_REF/refs\/tags\//}" >> "$GITHUB_OUTPUT"
           else
-              echo ::set-output name=baseURL::/unstable
+              echo "baseURL=/unstable" >> "$GITHUB_OUTPUT"
           fi
 
   build-openapi:
@@ -114,7 +114,7 @@ jobs:
       - name: "📥 Source checkout"
         uses: actions/checkout@v4
       - name: "➕ Setup Python"
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.9'
           cache: 'pip'
@@ -152,7 +152,7 @@ jobs:
               -o spec/server-server-api/api.json
           tar -czf openapi.tar.gz spec
       - name: "📤 Artifact upload"
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: openapi-artifact
           path: openapi.tar.gz
@@ -166,13 +166,15 @@ jobs:
       - name: "📥 Source checkout"
         uses: actions/checkout@v4
       - name: "➕ Setup Python"
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
       - name: "➕ Install towncrier"
         run: "pip install 'towncrier'"
       - name: "Generate changelog"
         run: ./scripts/generate-changelog.sh vUNSTABLE
       - name: "📤 Artifact upload"
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: changelog-artifact
           path: content/changelog/vUNSTABLE.md
@@ -185,11 +187,11 @@ jobs:
     if: ${{ always() }}
     steps:
       - name: "➕ Setup Node"
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: '18'
+          node-version: '20'
       - name: "➕ Setup Hugo"
-        uses: peaceiris/actions-hugo@16361eb4acea8698b220b76c0d4e84e1fd22c61d
+        uses: peaceiris/actions-hugo@75d2e84710de30f6ff7268e08f310b60ef14033f # v3.0.0
         with:
           hugo-version: '0.113.0'
           extended: true
@@ -201,7 +203,7 @@ jobs:
           npm run get-proposals
       - name: "📥 Download generated changelog"
         if: "needs.generate-changelog.result == 'success'"
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: changelog-artifact
           path: content/changelog
@@ -212,7 +214,7 @@ jobs:
       # https://spec.matrix.org/latest/client-server-api/api.json
       # Works for /unstable/ and /v1.1/ as well.
       - name: "📥 Spec definition download"
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: openapi-artifact
       - name: "📝 Unpack the OpenAPI definitions in the right location"
@@ -222,7 +224,7 @@ jobs:
       - name: "📦 Tarball creation"
         run: tar -czf spec.tar.gz spec
       - name: "📤 Artifact upload"
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: spec-artifact
           path: spec.tar.gz
@@ -236,7 +238,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: "📥 Fetch built spec"
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: spec-artifact
 
@@ -262,11 +264,11 @@ jobs:
     if: ${{ startsWith(github.ref, 'refs/tags/') }}
     steps:
       - name: "➕ Setup Node"
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: '18'
+          node-version: '20'
       - name: "➕ Setup Hugo"
-        uses: peaceiris/actions-hugo@16361eb4acea8698b220b76c0d4e84e1fd22c61d
+        uses: peaceiris/actions-hugo@75d2e84710de30f6ff7268e08f310b60ef14033f # v3.0.0
         with:
           # Cannot build the spec with Hugo 0.125.0 because of https://github.com/google/docsy/issues/1930
           hugo-version: '0.124.1'
@@ -284,7 +286,7 @@ jobs:
           hugo --config config.toml,historical.toml --baseURL "/${GITHUB_REF/refs\/tags\//}" -d "spec"
 
       - name: "📥 Spec definition download"
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: openapi-artifact
       - name: "📝 Unpack the OpenAPI definitions in the right location"
@@ -294,7 +296,7 @@ jobs:
       - name: "📦 Tarball creation"
         run: tar -czf spec-historical.tar.gz spec
       - name: "📤 Artifact upload"
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: spec-historical-artifact
           path: spec-historical.tar.gz

.github/workflows/netlify.yaml

@@ -32,10 +32,10 @@ jobs:
           pr_number=$(curl -H 'Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' "$pulls_uri" |
                jq -r '.[] | .number')
           echo "PR number: $pr_number"
-          echo "::set-output name=prnumber::$pr_number"
+          echo "prnumber=$pr_number" >> "$GITHUB_OUTPUT"
         
       - name: '📥 Download artifact'
-        uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e # v2.28.0
+        uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe # v3.1.4
         with:
           workflow: main.yaml
           run_id: ${{ github.event.workflow_run.id }}
@@ -46,8 +46,7 @@ jobs:
         
       - name: "📤 Deploy to Netlify"
         id: netlify
-        # v2.1.0
+        uses: nwtgck/actions-netlify@4cbaf4c08f1a7bfa537d6113472ef4424e4eb654 # v3.0.0
-        uses: nwtgck/actions-netlify@7a92f00dde8c92a5a9e8385ec2919775f7647352
         with:
           publish-dir: spec
           deploy-message: "Deploy from GitHub Actions"

.github/workflows/release.yaml

@@ -17,7 +17,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: 🔧 Yarn cache
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           cache: "yarn"
           cache-dependency-path: packages/npm/yarn.lock
@@ -34,7 +34,7 @@ jobs:
 
       - name: 🚀 Publish to npm
         id: npm-publish
-        uses: JS-DevTools/npm-publish@5a85faf05d2ade2d5b6682bfe5359915d5159c6c # v2.2.1
+        uses: JS-DevTools/npm-publish@19c28f1ef146469e409470805ea4279d47c3d35c # v3.1.1
         with:
           token: ${{ secrets.NPM_TOKEN }}
           package: packages/npm

.github/workflows/spell-check.yaml

@@ -14,6 +14,6 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Check spelling of proposals
-      uses: crate-ci/typos@ff3f309513469397e1094520fb7a054e057589e1
+      uses: crate-ci/typos@f2c1f08a7b3c1b96050cb786baaa2a94797bdb7d # v1.20.10
       with:
-        config: ${{github.workspace}}/.github/_typos.toml
+        config: ${{github.workspace}}/.github/_typos.toml

changelogs/client_server/newsfragments/1808.clarification

@@ -0,0 +1 @@
+Deprecate authentication via a query string, as per [MSC4126](https://github.com/matrix-org/matrix-spec-proposals/issues/4126).

changelogs/identity_service/newsfragments/1808.clarification

@@ -0,0 +1 @@
+Deprecate authentication via a query string, as per [MSC4126](https://github.com/matrix-org/matrix-spec-proposals/issues/4126).

changelogs/internal/newsfragments/1796.clarification

@@ -0,0 +1 @@
+Add support for pattern formats for `patternProperties`.

changelogs/internal/newsfragments/1799.clarification

@@ -0,0 +1 @@
+Fix anchors for schemas under `oneOf`.

changelogs/internal/newsfragments/1802.clarification

@@ -0,0 +1 @@
+Add anchors in `definition` shortcode.

changelogs/internal/newsfragments/1803.clarification

@@ -0,0 +1 @@
+Update most CI actions.

changelogs/internal/newsfragments/1804.clarification

@@ -0,0 +1 @@
+Update typos CI action.

changelogs/internal/newsfragments/1805.clarification

@@ -0,0 +1 @@
+Set python version for the Towncrier CI job.

changelogs/internal/newsfragments/1806.clarification

@@ -0,0 +1 @@
+Replace `set-output` with environment files in CI.

changelogs/internal/newsfragments/1809.clarification

@@ -0,0 +1 @@
+Render response headers.

content/client-server-api/_index.md

@@ -390,8 +390,7 @@ specify parameter values. The flow for this method is as follows:
 ## Client Authentication
 
 Most API endpoints require the user to identify themselves by presenting
-previously obtained credentials in the form of an `access_token` query
+previously obtained credentials in the form of an access token.
-parameter or through an Authorization Header of `Bearer $access_token`.
 An access token is typically obtained via the [Login](#login) or
 [Registration](#account-registration-and-management) processes. Access tokens
 can expire; a new access token can be generated by using a refresh token.
@@ -405,16 +404,19 @@ investigate [macaroons](http://research.google.com/pubs/pub41892.html).
 
 ### Using access tokens
 
-Access tokens may be provided in two ways, both of which the homeserver
+Access tokens may be provided via a request header, using the Authentication
-MUST support:
+Bearer scheme: `Authorization: Bearer TheTokenHere`.
 
-1.  Via a query string parameter, `access_token=TheTokenHere`.
+Clients may alternatively provide the access token via a query string parameter:
-2.  Via a request header, `Authorization: Bearer TheTokenHere`.
+`access_token=TheTokenHere`. This method is deprecated to prevent the access
+token being leaked in access/HTTP logs and SHOULD NOT be used by clients.
 
-Clients are encouraged to use the `Authorization` header where possible
+Homeservers MUST support both methods.
-to prevent the access token being leaked in access/HTTP logs. The query
+
-string should only be used in cases where the `Authorization` header is
+{{% boxes/note %}}
-inaccessible for the client.
+{{% changed-in v="1.11" %}}
+Sending the access token as a query string parameter is now deprecated.
+{{% /boxes/note %}}
 
 When credentials are required but missing or invalid, the HTTP call will
 return with a status of 401 and the error code, `M_MISSING_TOKEN` or

content/identity-service-api.md

@@ -162,15 +162,20 @@ of access tokens to authenticate users. The access tokens provided by an
 Identity Server cannot be used to authenticate Client-Server API
 requests.
 
-An access token is provided to an endpoint in one of two ways:
+Access tokens may be provided via a request header, using the
+Authentication Bearer scheme: `Authorization: Bearer TheTokenHere`.
 
-1.  Via a query string parameter, `access_token=TheTokenHere`.
+Clients may alternatively provide the access token via a query string
-2.  Via a request header, `Authorization: Bearer TheTokenHere`.
+parameter: `access_token=TheTokenHere`. This method is deprecated to
+prevent the access token being leaked in access/HTTP logs and SHOULD NOT
+be used by clients.
 
-Clients are encouraged to the use the `Authorization` header where
+Identity Servers MUST support both methods.
-possible to prevent the access token being leaked in access/HTTP logs.
+
-The query string should only be used in cases where the `Authorization`
+{{% boxes/note %}}
-header is inaccessible for the client.
+{{% changed-in v="1.11" %}}
+Sending the access token as a query string parameter is now deprecated.
+{{% /boxes/note %}}
 
 When credentials are required but missing or invalid, the HTTP call will
 return with a status of 401 and the error code `M_UNAUTHORIZED`.

data/api/client-server/definitions/security.yaml

@@ -14,7 +14,7 @@
 accessTokenQuery:
   type: apiKey
   description: |-
-    The `access_token` returned by a call to `/login` or `/register`, as a query
+    **Deprecated.** The `access_token` returned by a call to `/login` or `/register`, as a query
     parameter.
 
     It can also be the `as_token` of an application service.
@@ -33,7 +33,8 @@ accessTokenBearer:
 appserviceAccessTokenQuery:
   type: apiKey
   description: |-
-    The `as_token` of an application service, as a query parameter.
+    **Deprecated.** The `as_token` of an application service, as a query
+    parameter.
   name: access_token
   in: query
 appserviceAccessTokenBearer:

data/api/identity/definitions/security.yaml

@@ -14,7 +14,8 @@
 accessTokenQuery:
   type: apiKey
   description: |-
-    The `access_token` returned by a call to `/register`, as a query parameter.
+    **Deprecated.** The `access_token` returned by a call to `/register`, as a
+    query parameter.
   name: access_token
   in: query
 accessTokenBearer:

data/custom-formats.yaml

@@ -0,0 +1,33 @@
+# Copyright 2024 Commaille Kévin
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file contains the list of custom formats supported for the `format` key
+# and the `x-pattern-format` extension (see `openapi_extensions.md` for more
+# details).
+#
+# Each entry must use the `mx-` prefix and have the form:
+#
+# mx-custom-key:
+#   title: The title rendered in the specification
+#   url: /url/to#definition
+
+mx-user-id:
+  title: User ID
+  url: /appendices#user-identifiers
+  # regex: "^@"
+
+mx-event-id:
+  title: Event ID
+  url: /appendices#event-ids
+  # regex: "^\\$"

data/event-schemas/schema/m.ignored_user_list.yaml

@@ -17,7 +17,7 @@ properties:
           "^@":
             type: "object"
             description: "An empty object for future enhancement"
-            x-pattern: "$USER_ID"
+            x-pattern-format: "mx-user-id"
     required:
       - ignored_users
   type:

data/event-schemas/schema/m.receipt.yaml

@@ -16,7 +16,7 @@ properties:
         patternProperties:
             "^\\$":
                 type: object
-                x-pattern: "$EVENT_ID"
+                x-pattern-format: "mx-event-id"
                 title: Event Receipts
                 description: |-
                     The collection of receipts for this event ID.
@@ -34,7 +34,7 @@ properties:
                                 description: |-
                                     The mapping of user ID to receipt. The user ID is the
                                     entity who sent this receipt.
-                                x-pattern: "$USER_ID"
+                                x-pattern-format: "mx-user-id"
                                 properties:
                                     ts:
                                         type: integer

layouts/partials/json-schema/resolve-additional-types.html

@@ -165,6 +165,7 @@
      * (https://json-schema.org/understanding-json-schema/reference/combining.html#oneof)
      */
     {{ if $this_object.oneOf }}
+        {{ $updated_items := slice }}
         {{ range $idx, $item := $this_object.oneOf }}
             {{ $res := partial "get-additional-objects" (dict
                 "this_object" $item
@@ -173,7 +174,11 @@
                 "name" (printf "%s.oneOf[%d]" $name $idx)
             ) }}
             {{ $all_objects = $res.objects }}
+            {{ $updated_items = $updated_items | append $res.schema }}
         {{ end }}
+
+        /* Update the top-level schema with the updated subschemas for the items */
+        {{ $this_object = merge $this_object (dict "oneOf" $updated_items) }}
     {{ end }}
 
     {{ return (dict

layouts/partials/openapi/render-object-table.html

@@ -179,17 +179,45 @@
         {{/*
             If the property uses `patternProperties` to describe its
             internal structure, handle this with a bit of recursion.
-            Note that we ignore the pattern as the current definitions
+            Types are grouped by pattern format. Note that we ignore
+            patterns without a format as the current definitions
             always have a single pattern, but we might need to handle
             them later to differentiate schemas according to patterns.
         */}}
-        {{ $types := slice }}
 
-        {{ range $pattern, $schema := .patternProperties}}
+        {{/*
-            {{ $types = $types | append (partial "property-type" $schema) }}
+            Construct a map from format ID to the type string of the format.
+        */}}
+        {{ $formatMap := newScratch }}
+
+        {{ range $pattern, $schema := .patternProperties }}
+            {{ $formatId := or (index $schema "x-pattern-format") "string" }}
+
+            {{ if $formatMap.Get $formatId }}
+                {{ errorf "'%s' pattern format is defined more than once for the same property" $formatId }}
+            {{ end }}
+
+            {{ $formatMap.Set $formatId (partial "property-type" $schema) }}
         {{ end }}
 
-        {{ $type = delimit (slice "{string: " (delimit $types "|") "}" ) "" }}
+        {{/* First generate the type string for each format. */}}
+        {{ $types := slice }}
+        {{ range $formatId, $formatType := $formatMap.Values }}
+            {{ $formatKey := "string" }}
+            {{ if ne $formatId "string" }}
+                {{ with index site.Data "custom-formats" $formatId }}
+                    {{ $formatKey = printf "<a href=\"%s\">%s</a>" (htmlEscape .url) (htmlEscape .title) }}
+                {{ else }}
+                    {{ errorf "Unsupported value for `x-pattern-format`: %s" $formatId }}
+                {{ end }}
+            {{ end }}
+
+            {{ $formatString := printf "{%s: %s}" $formatKey $formatType }}
+            {{ $types = $types | append $formatString }}
+        {{ end }}
+
+        {{/* Then join all the formats. */}}
+        {{ $type = delimit $types "|" }}
     {{ else if .title }}
         {{/*
             No properties, so there won't be a separate table. We use the title

layouts/partials/openapi/render-responses.html

@@ -38,6 +38,24 @@
 {{ range $code, $response := $responses }}
     {{ if $response.content }}
 <h3>{{$code}} response</h3>
+        {{/* Display defined headers */}}
+        {{ if $response.headers }}
+            {{/* build a dict mapping from name->schema, which render-object-table expects */}}
+            {{ $headers_dict := dict }}
+            {{ range $header_name,$header_props := $response.headers }}
+                {{/*
+                    merge the schema at the same level as the rest of the other fields because that is
+                    what `render-object-table` expects. Put the schema first so examples in it are
+                    overwritten.
+                */}}
+                {{ $header_schema := merge $header_props.schema $header_props }}
+                {{ $headers_dict = merge $headers_dict (dict $header_name $header_schema )}}
+            {{ end }}
+
+            {{/* and render the headers */}}
+            {{ partial "openapi/render-object-table" (dict "title" "Headers" "properties" $headers_dict) }}
+        {{ end }}
+
         {{/*
             A response can have several content types.
         */}}

layouts/shortcodes/definition.html

@@ -26,7 +26,9 @@
 {{ $definition = partial "json-schema/resolve-refs" (dict "schema" $definition "path" $path) }}
 {{ $definition = partial "json-schema/resolve-allof" $definition }}
 
-<section class="rendered-data definition" id="definition-{{ anchorize $definition.title }}">
+{{ $anchor_base := printf "definition-%s" (anchorize $definition.title) }}
+
+<section class="rendered-data definition" id="{{ $anchor_base }}">
 
 <details {{ if not $compact }}open{{ end }}>
 <summary>
@@ -46,7 +48,11 @@
 {{ $definition.description | markdownify }}
 
 
-{{ $additional_types := partial "json-schema/resolve-additional-types" (dict "schema" $definition "name" (printf "\"%s\"" $path)) }}
+{{ $additional_types := partial "json-schema/resolve-additional-types" (dict
+    "schema" $definition
+    "anchor_base" $anchor_base
+    "name" (printf "\"%s\"" $path))
+}}
 
 {{ range $additional_types }}
     {{ partial "openapi/render-object-table" . }}

openapi_extensions.md

@@ -31,3 +31,13 @@ particular Matrix specification versions.
 
 Although the OpenAPI/JSON Schema specs only allow to use `$ref` to reference a
 whole example, we use it to compose examples from other examples.
+
+## Custom `x-pattern-format` key and custom formats 
+
+In JSON Schema, [`format`](https://json-schema.org/understanding-json-schema/reference/string#format)
+is a property to convey semantic information about a schema. We define
+`x-pattern-format` as a key on the schemas under `patternProperties` with the
+same use as `format`, but that applies to the pattern of the property. We also
+define custom values for formats with the `mx-` prefix in
+`data/custom-formats.yaml`. Those values are recognized in the rendered
+specification and link to the definition of the format.