Merge fe6c97f498 into 70c7d59caa

Some clarifications to the release process doc, particularly in view of #2275.

.github/workflows/main.yml

@@ -195,6 +195,8 @@ jobs:
     needs: [calculate-baseurl, build-openapi, generate-changelog]
     # run even if generate-changelog was skipped
     if: ${{ always() }}
+    env:
+      baseURL: "${{ needs.calculate-baseurl.outputs.baseURL }}"
     steps:
       - name: "➕ Setup Node"
         uses: actions/setup-node@v4
@@ -217,8 +219,10 @@ jobs:
         with:
           name: changelog-artifact
           path: content/changelog
+
       - name: "⚙️ hugo"
-        run: hugo --baseURL "${{ needs.calculate-baseurl.outputs.baseURL }}" -d "spec"
+        run: hugo --baseURL "${baseURL}" -d "spec${baseURL}"
+
       # We manually unpack the spec OpenAPI definition JSON to the website tree
       # to make it available to the world in a canonical place:
       # https://spec.matrix.org/latest/client-server-api/api.json
@@ -229,10 +233,13 @@ jobs:
           name: openapi-artifact
       - name: "📝 Unpack the OpenAPI definitions in the right location"
         run: |
-          tar -xzf openapi.tar.gz
+          tar -C "spec${baseURL}" --strip-components=1 -xzf openapi.tar.gz
 
       - name: "📦 Tarball creation"
-        run: tar -czf spec.tar.gz spec
+        run: |
+          cd spec
+          tar -czf ../spec.tar.gz *
+
       - name: "📤 Artifact upload"
         uses: actions/upload-artifact@v4
         with:
@@ -261,14 +268,9 @@ jobs:
           name: spec-artifact
 
       - name: "📝 Unpack the spec"
-        # we have to unpack it into the right path given the baseurl, so that the
-        # links are correct.
-        # eg if baseurl is `/unstable`, we want to put the site in `spec/unstable`.
         run: |
-          mkdir -p "spec${baseURL}"
+          mkdir spec
-          tar -C "spec${baseURL}" --strip-components=1 -xvzf spec.tar.gz
+          tar -C spec -xvzf spec.tar.gz
-        env:
-          baseURL: "${{ needs.calculate-baseurl.outputs.baseURL }}"
 
       - name: "Run htmltest"
         uses: wjdp/htmltest-action@master
@@ -278,8 +280,10 @@ jobs:
   build-historical-spec:
     name: "📖 Build the historical backup spec"
     runs-on: ubuntu-latest
-    needs: [build-openapi]
+    needs: [calculate-baseurl, build-openapi]
     if: ${{ startsWith(github.ref, 'refs/tags/') }}
+    env:
+      baseURL: "${{ needs.calculate-baseurl.outputs.baseURL }}"
     steps:
       - name: "➕ Setup Node"
         uses: actions/setup-node@v4
@@ -299,9 +303,8 @@ jobs:
       - name: "⚙️ hugo"
         env:
           HUGO_PARAMS_VERSION_STATUS: "historical"
-        # Create a baseURL like `/v1.2` out of the `v1.2` tag
         run: |
-          hugo --baseURL "/${GITHUB_REF/refs\/tags\//}" -d "spec"
+          hugo --baseURL "${baseURL}" -d "spec${baseURL}"
 
       - name: "📥 Spec definition download"
         uses: actions/download-artifact@v4
@@ -309,10 +312,12 @@ jobs:
           name: openapi-artifact
       - name: "📝 Unpack the OpenAPI definitions in the right location"
         run: |
-          tar -xzf openapi.tar.gz
+          tar -C "spec${baseURL}" --strip-components=1 -xzf openapi.tar.gz
 
       - name: "📦 Tarball creation"
-        run: tar -czf spec-historical.tar.gz spec
+        run: |
+          cd spec
+          tar -czf ../spec-historical.tar.gz *
 
       - name: "📤 Artifact upload"
         uses: actions/upload-artifact@v4

.github/workflows/netlify.yaml

@@ -45,7 +45,9 @@ jobs:
           name: spec-artifact
 
       - name: "📦 Extract Artifacts"
-        run: tar -xzvf spec.tar.gz && rm spec.tar.gz
+        run: |
+          mkdir spec
+          tar -C spec -xzvf spec.tar.gz && rm spec.tar.gz
         
       - name: "📤 Deploy to Netlify"
         id: netlify

changelogs/internal/newsfragments/2222.clarification

@@ -0,0 +1 @@
+Clarify vendor prefixing requirements.

changelogs/internal/newsfragments/2276.feature

@@ -0,0 +1 @@
+Include the spec release version in the filenames in the tarballs generated by CI.

changelogs/internal/newsfragments/2289.clarification

@@ -0,0 +1 @@
+Updates to the release documentation.

changelogs/server_server/newsfragments/2284.clarification

@@ -0,0 +1 @@
+Specified input validation for PDUs passed to federation membership endpoints.

changelogs/server_server/newsfragments/2288.clarification

@@ -0,0 +1 @@
+Specify that callers of `/_matrix/federation/v1/openid/userinfo` must validate the returned user ID.

content/proposals.md

@@ -408,41 +408,9 @@ development or testing data.
 that a particular MSC works) do not have to follow this process.
 
 1.  Have an idea for a feature.
-1.  Implement the feature using unstable endpoints, vendor prefixes, and
+1.  Implement the feature using [unstable endpoints, vendor prefixes, and
-    unstable feature flags as appropriate.
+    unstable feature flags](#unstable-endpoints-features-and-vendor-prefixes)
-    -   When using unstable endpoints, they MUST include a vendor
+    as appropriate.
-        prefix. For example:
-        `/_matrix/client/unstable/com.example/login`. Vendor prefixes
-        throughout Matrix always use the Java package naming convention.
-        The MSC for the feature should identify which preferred vendor
-        prefix is to be used by early adopters.
-    -   Note that unstable namespaces do not automatically inherit
-        endpoints from stable namespaces: for example, the fact that
-        `/_matrix/client/r0/sync` exists does not imply that
-        `/_matrix/client/unstable/com.example/sync` exists.
-    -   If the client needs to be sure the server supports the feature,
-        an unstable feature flag that MUST be vendor prefixed is to be
-        used. This kind of flag shows up in the `unstable_features`
-        section of `/versions` as, for example, `com.example.new_login`.
-        The MSC for the feature should identify which preferred feature
-        flag is to be used by early adopters.
-    -   When using this approach correctly, the implementation can
-        ship/release the feature at any time, so long as the
-        implementation is able to accept the technical debt that results
-        from needing to provide adequate backwards and forwards
-        compatibility. The implementation MUST support the flag (and
-        server-side implementation) disappearing and be generally safe
-        for users. Note that implementations early in the MSC review
-        process may also be required to provide backwards compatibility
-        with earlier editions of the proposal.
-    -   If the implementation cannot support the technical debt (or if
-        it's impossible to provide forwards/backwards compatibility -
-        e.g. a user authentication change which can't be safely rolled
-        back), the implementation should not attempt to implement the
-        feature and should instead wait for a spec release.
-    -   If at any point after early release, the idea changes in a
-        backwards-incompatible way, the feature flag should also change
-        so that implementations can adapt as needed.
 1.  In parallel, or ahead of implementation, open an MSC and solicit
     review per above.
 1.  Before FCP can be called, the Spec Core Team will require evidence
@@ -452,10 +420,7 @@ that a particular MSC works) do not have to follow this process.
     forwards/backwards compatibility concerns mentioned here.
 1.  The FCP process is completed, and assuming nothing is flagged the
     MSC lands.
-1.  Implementations can now switch to using stable prefixes
+1.  Implementations can now switch to using stable prefixes, assuming that the change
-    (for example, for an endpoint, moving from
-    `/unstable/org.matrix.mscxxxx/frobnicate`
-    to `/v1/frobnicate`), assuming that the change
     is backwards compatible with older implementations. In the rare occasion
     where backwards compatibility is not possible without a new spec release,
     implementations should continue to use unstable prefixes.
@@ -471,13 +436,6 @@ that a particular MSC works) do not have to follow this process.
     started supporting the new spec release, some noise should be raised
     in the general direction of the implementation.
 
-{{% boxes/note %}}
-MSCs MUST still describe what the stable endpoints/feature looks like
-with a note towards the bottom for what the unstable feature
-flag/prefixes are. For example, an MSC would propose `/_matrix/client/r0/new/endpoint`, not `/_matrix/client/unstable/
-com.example/new/endpoint`.
-{{% /boxes/note %}}
-
 In summary:
 
 -   Implementations MUST NOT use stable endpoints before the MSC has
@@ -489,14 +447,90 @@ In summary:
 -   Implementations SHOULD be wary of the technical debt they are
     incurring by moving faster than the spec.
 -   The vendor prefix is chosen by the developer of the feature, using
-    the Java package naming convention. The foundation's preferred
+    the Java package naming convention.
-    vendor prefix is `org.matrix`.
 -   The vendor prefixes, unstable feature flags, and unstable endpoints
     should be included in the MSC, though the MSC MUST be written in a
     way that proposes new stable endpoints. Typically this is solved by
     a small table at the bottom mapping the various values from stable
     to unstable.
 
+#### Unstable endpoints, features and vendor prefixes
+
+Unstable endpoints MUST use `/unstable` as the endpoint version and a
+vendor prefix in Java package naming format. For example:
+`/_matrix/client/unstable/com.example.mscxxxx/login`.
+
+{{% boxes/note %}}
+Proposal authors operating with a Matrix.org Foundation mandate SHOULD use
+a vendor prefix within the `org.matrix` namespace. This namespace is otherwise
+restricted. Authors who don't own a domain MAY use the `io.github` namespace
+instead.
+{{% /boxes/note %}}
+
+Note that unstable namespaces do not automatically inherit endpoints from
+stable namespaces: for example, the fact that `/_matrix/client/v3/sync`
+exists does not imply that `/_matrix/client/unstable/com.example.mscxxxx/sync`
+exists.
+
+Vendor prefixes MUST also be used for:
+
+-   New parameters on existing endpoints. For example:
+    `/_matrix/client/v3/publicRooms?com.example.mscxxxx.ordered_by=member_count`.
+-   New properties in existing JSON objects. For example:
+
+    ```json
+    {
+      "avatar_url": "mxc://matrix.org/SDGdghriugerRg",
+      "displayname": "Alice Margatroid",
+      "com.example.mscxxxx.phone": [{
+        "type": "landline",
+        "number": "+1-206-555-7000"
+      }],
+      ...
+    }
+    ```
+
+-   New values for existing parameters or properties. For example:
+
+    ```json
+    {
+      "errcode": "COM.EXAMPLE.MSCXXXX.M_INVALID_EMAIL",
+      "error": "The email address you provided is invalid."
+    }
+    ```
+
+If the client needs to be sure the server supports the feature, an
+unstable feature flag that MUST also be vendor prefixed is to be used.
+This flag shows up in the `unstable_features` section of
+[`/_matrix/client/versions`](/client-server-api/#get_matrixclientversions)
+as, for example, `com.example.mscxxxx.new_login`.
+
+{{% boxes/note %}}
+MSCs MUST still describe what the stable endpoints/feature looks like
+with a note towards the bottom for what the unstable feature
+flag/prefixes are. For example, an MSC would propose `/_matrix/client/v1/new/endpoint`,
+not `/_matrix/client/unstable/com.example.mscxxxx/new/endpoint`.
+{{% /boxes/note %}}
+
+When using this approach correctly, the implementation can release
+the feature at any time, so long as the implementation is able to
+accept the technical debt that results from needing to provide
+adequate backwards and forwards compatibility. The implementation
+MUST support the flag (and server-side implementation) disappearing
+and be generally safe for users. Note that implementations early in
+the MSC review process may also be required to provide backwards
+compatibility with earlier editions of the proposal.
+
+If the implementation cannot support the technical debt (or if it's
+impossible to provide forwards/backwards compatibility - e.g. a user
+authentication change which can't be safely rolled back), the
+implementation should not attempt to implement the feature and should
+instead wait for a spec release.
+
+If at any point after early release, the idea changes in a
+backwards-incompatible way, the feature flag should also change so
+that implementations can adapt as needed.
+
 ### Placeholder MSCs
 
 Some proposals may contain security-sensitive or private context which can't be

data/api/server-server/invites-v1.yaml

@@ -172,6 +172,17 @@ paths:
                   }
         "400":
           description: |-
+            The `M_INVALID_PARAM` error code is used to indicate one or more of the following:
+
+            * The invite event fails a [signature check](/server-server-api/#validating-hashes-and-signatures-on-received-events).
+            * The event type is not `m.room.member`.
+            * The `membership` field inside the event content is not `invite`.
+            * The event sender is not a user ID on the origin server.
+            * The `state_key` is not a user ID on the receiving server.
+
+            Servers MUST apply the validation above to the invite event before
+            signing it regardless of room version.
+
             The `M_MISSING_PARAM` error code is used to indicate one or more of
             the following:
 
@@ -186,9 +197,9 @@ paths:
             Servers MAY apply the validation above to room versions 1 through 11,
             and SHOULD apply the validation above to all other room versions.
 
-            If `M_MISSING_PARAM` is returned and the request is associated with a
+            If `M_MISSING_PARAM` or `M_INVALID_PARAM` is returned and the request
-            Client-Server API request, the Client-Server API request SHOULD fail
+            is associated with a Client-Server API request, the Client-Server API
-            with a 5xx error rather than being passed through.
+            request SHOULD fail with a 5xx error rather than being passed through.
           content:
             application/json:
               schema:

data/api/server-server/invites-v2.yaml

@@ -154,6 +154,17 @@ paths:
             The error should be passed through to clients so that they
             may give better feedback to users.
 
+            The `M_INVALID_PARAM` error code is used to indicate one or more of the following:
+
+            * The invite event fails a [signature check](/server-server-api/#validating-hashes-and-signatures-on-received-events).
+            * The event type is not `m.room.member`.
+            * The `membership` field inside the event content is not `invite`.
+            * The event sender is not a user ID on the origin server.
+            * The `state_key` is not a user ID on the receiving server.
+
+            Servers MUST apply the validation above to the invite event before
+            signing it regardless of room version.
+
             The `M_MISSING_PARAM` error code is used to indicate one or more of
             the following:
 
@@ -168,9 +179,9 @@ paths:
             Servers MAY apply the validation above to room versions 1 through 11,
             and SHOULD apply the validation above to all other room versions.
 
-            If `M_MISSING_PARAM` is returned and the request is associated with a
+            If `M_MISSING_PARAM` or `M_INVALID_PARAM` is returned and the request
-            Client-Server API request, the Client-Server API request SHOULD fail
+            is associated with a Client-Server API request, the Client-Server API
-            with a 5xx error rather than being passed through.
+            request SHOULD fail with a 5xx error rather than being passed through.
           content:
             application/json:
               schema:

data/api/server-server/joins-v1.yaml

@@ -36,7 +36,7 @@ paths:
             type: string
         - in: path
           name: userId
-          description: The user ID the join event will be for.
+          description: The user ID the join event will be for. This MUST be a user ID on the origin server.
           required: true
           example: "@someone:example.org"
           schema:
@@ -388,6 +388,43 @@ paths:
                       }
                     }
                   ]
+        "400":
+          description: |-
+            The request is invalid in some way.
+
+            The `M_INVALID_PARAM` error code is used to indicate one or more of the following:
+
+            * The join event fails a [signature check](/server-server-api/#validating-hashes-and-signatures-on-received-events).
+            * The event type is not `m.room.member`.
+            * The `membership` field inside the event content is not `join`.
+            * The event sender is not a user ID on the origin server.
+            * The `state_key` is not equal to the `sender`.
+
+            Servers MUST apply the validation above to the join event.
+          content:
+            application/json:
+              schema:
+                $ref: ../client-server/definitions/errors/error.yaml
+              examples:
+                response:
+                  value: {
+                    "errcode": "M_INVALID_PARAM",
+                    "error": "Not a join event."
+                  }
+        "403":
+          description: |-
+            The room that the joining server is attempting to join does not permit the user
+            to join.
+          content:
+            application/json:
+              schema:
+                $ref: ../client-server/definitions/errors/error.yaml
+              examples:
+                response:
+                  value: {
+                    "errcode": "M_FORBIDDEN",
+                    "error": "You are not invited to this room"
+                  }
 servers:
   - url: "{protocol}://{hostname}{basePath}"
     variables:

data/api/server-server/joins-v2.yaml

@@ -247,6 +247,16 @@ paths:
             The error should be passed through to clients so that they
             may give better feedback to users.
 
+            The `M_INVALID_PARAM` error code is used to indicate one or more of the following:
+
+            * The join event fails a [signature check](/server-server-api/#validating-hashes-and-signatures-on-received-events).
+            * The event type is not `m.room.member`.
+            * The `membership` field inside the event content is not `join`.
+            * The event sender is not a user ID on the origin server.
+            * The `state_key` is not equal to the `sender`.
+
+            Servers MUST apply the validation above to the join event.
+
             New in `v1.2`, the following error conditions might happen:
 
             If the room is [restricted](/client-server-api/#restricted-rooms)

data/api/server-server/knocks.yaml

@@ -36,7 +36,7 @@ paths:
             type: string
         - in: path
           name: userId
-          description: The user ID the knock event will be for.
+          description: The user ID the knock event will be for. This MUST be a user ID on the origin server.
           required: true
           example: "@someone:example.org"
           schema:
@@ -330,6 +330,27 @@ paths:
                       "$ref": "./examples/invite_or_knock_state.json"
                     }
                   }
+        "400":
+          description: |-
+            The `M_INVALID_PARAM` error code is used to indicate one or more of the following:
+
+            * The knock event fails a [signature check](/server-server-api/#validating-hashes-and-signatures-on-received-events).
+            * The event type is not `m.room.member`.
+            * The `membership` field inside the event content is not `knock`.
+            * The event sender is not a user ID on the origin server.
+            * The `state_key` is not equal to the `sender`.
+
+            Servers MUST apply the validation above to the knock event.
+          content:
+            application/json:
+              schema:
+                $ref: ../client-server/definitions/errors/error.yaml
+              examples:
+                response:
+                  value: {
+                    "errcode": "M_INVALID_PARAM",
+                    "error": "Not a knock event."
+                  }
         "403":
           description: |-
             The knocking server or user is not permitted to knock on the room, such as when the

data/api/server-server/leaving-v1.yaml

@@ -36,7 +36,7 @@ paths:
             type: string
         - in: path
           name: userId
-          description: The user ID the leave event will be for.
+          description: The user ID the leave event will be for. This MUST be a user ID on the origin server.
           required: true
           example: "@someone:example.org"
           schema:
@@ -249,6 +249,27 @@ paths:
                     200,
                     {}
                   ]
+        "400":
+          description: |-
+            The `M_INVALID_PARAM` error code is used to indicate one or more of the following:
+
+            * The leave event fails a [signature check](/server-server-api/#validating-hashes-and-signatures-on-received-events).
+            * The event type is not `m.room.member`.
+            * The `membership` field inside the event content is not `leave`.
+            * The event sender is not a user ID on the origin server.
+            * The `state_key` is not equal to the `sender`.
+
+            Servers MUST apply the validation above to the leave event.
+          content:
+            application/json:
+              schema:
+                $ref: ../client-server/definitions/errors/error.yaml
+              examples:
+                response:
+                  value: {
+                    "errcode": "M_INVALID_PARAM",
+                    "error": "Not a leave event."
+                  }
 servers:
   - url: "{protocol}://{hostname}{basePath}"
     variables:

data/api/server-server/leaving-v2.yaml

@@ -134,6 +134,27 @@ paths:
               examples:
                 response:
                   value: {}
+        "400":
+          description: |-
+            The `M_INVALID_PARAM` error code is used to indicate one or more of the following:
+
+            * The leave event fails a [signature check](/server-server-api/#validating-hashes-and-signatures-on-received-events).
+            * The event type is not `m.room.member`.
+            * The `membership` field inside the event content is not `leave`.
+            * The event sender is not a user ID on the origin server.
+            * The `state_key` is not equal to the `sender`.
+
+            Servers MUST apply the validation above to the leave event.
+          content:
+            application/json:
+              schema:
+                $ref: ../client-server/definitions/errors/error.yaml
+              examples:
+                response:
+                  value: {
+                    "errcode": "M_INVALID_PARAM",
+                    "error": "Not a leave event."
+                  }
 servers:
   - url: "{protocol}://{hostname}{basePath}"
     variables:

data/api/server-server/openid.yaml

@@ -43,7 +43,12 @@ paths:
                 properties:
                   sub:
                     type: string
-                    description: The Matrix User ID who generated the token.
+                    description: |
+                      The Matrix User ID who generated the token.
+
+                      The caller MUST validate that the returned user ID is on the server they
+                      called (i.e. if you make a request to example.com and it returns
+                      `@alice:matrix.org`, the result is invalid).
                     example: "@alice:example.com"
                 required:
                   - sub

meta/releasing.md

@@ -50,11 +50,6 @@ First, can we even release the spec? This stage is mostly preparation work neede
 to ensure a consistent and reliable specification.
 
 1. Ensure `main` is committed with all the spec changes you expect to be there.
-2. Review the changelog to look for typos, wording inconsistencies, or lines which
-   can be merged. For example, "Fix typos" and "Fix spelling" can be condensed to
-   "Fix various typos throughout the specification".
-3. Do a quick skim to ensure changelogs reference the MSCs which brought the changes
-   in. They should be linked to the GitHub MSC PR (not the markdown document).
 
 ## The release
 
@@ -79,20 +74,24 @@ release.
    2. Run `./scripts/generate-changelog.sh v1.2` (using the correct version number).
       The script will use the current date. If that date is wrong, correct the document
       by using the same `YYYY-MM-DD` date format.
-   3. Commit the result.
+   3. Review the changelog to look for typos, wording inconsistencies, or lines which
+      can be merged. For example, "Fix typos" and "Fix spelling" can be condensed to
+      "Fix various typos throughout the specification".
+   4. Do a quick skim to ensure changelogs reference the MSCs which brought the changes
+      in. They should be linked to the GitHub MSC PR (not the markdown document).
+   5. Commit the result.
+   6. Now is a good time to have someone else review the changelog.
 5. Tag the branch with the spec release with a format of `v1.2` (if releasing Matrix 1.2).
 6. Push the release branch and the tag.
 7. GitHub Actions will run its build steps. Wait until these are successful. If fixes
    need to be made to repair the pipeline or spec build, delete and re-tag the release.
    You may need to fix up the changelog file by hand in this case.
-8. Check out and fast-forward `main` to the release branch.
+8. GitHub Actions should have drafted a release based on the new tag. Find it
-9. Create a new release on GitHub from the newly created tag.
+   at https://github.com/matrix-org/matrix-spec/releases.
-   * The title should be just "v1.2" (for example).
+   1. Double-check the generated release notes, and check that `spec-artifact.zip` and
-   * The description should be a copy/paste of the changelog. The generated changelog
+      `spec-historical-artifact.zip` are both attached to the draft release.
-     will be at `content/changelog/v1.2.md` - copy/paste verbatim.
+   2. Publish the draft release.
-   * Upload the artifacts of the GitHub Actions build for the release to the GitHub
+9. Check out and fast-forward `main` to the release branch.
-     release as artifacts themselves. This should be the tarball that will be deployed
-     to spec.matrix.org.
 10. Commit a reversion to `params.version` of `./config/_default/hugo.toml` on `main`:
     ```toml
     [params.version]
@@ -103,7 +102,8 @@ release.
     ```
 11. Push pending commits and ensure the unstable spec updates accordingly from the
     GitHub Actions pipeline.
-12. Deploy the release on the webserver. See internal wiki.
+12. Deploy the release on the webserver. See "Spec release process" in the
+    internal handbook.
 
 ## Patching a release