Merge 339ea6be12 into fe46e0c363

Signed-off-by: Kévin Commaille <zecakeh@tedomum.fr>

.github/workflows/main.yml

@@ -2,6 +2,7 @@ name: "Spec"
 
 env:
   HUGO_VERSION: 0.139.0
+  PYTHON_VERSION: 3.13
 
 on:
   push:
@@ -40,7 +41,7 @@ jobs:
       - name: "➕ Setup Python"
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip'
           cache-dependency-path: scripts/requirements.txt
       - name: "➕ Install dependencies"
@@ -59,7 +60,7 @@ jobs:
       - name: "➕ Setup Python"
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip'
           cache-dependency-path: scripts/requirements.txt
       - name: "➕ Install dependencies"
@@ -78,7 +79,7 @@ jobs:
       - name: "➕ Setup Python"
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip'
           cache-dependency-path: scripts/requirements.txt
       - name: "➕ Install dependencies"
@@ -120,7 +121,7 @@ jobs:
       - name: "➕ Setup Python"
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip'
           cache-dependency-path: scripts/requirements.txt
       - name: "➕ Install dependencies"
@@ -172,7 +173,7 @@ jobs:
       - name: "➕ Setup Python"
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: ${{ env.PYTHON_VERSION }}
       - name: "➕ Install towncrier"
         run: "pip install 'towncrier'"
       - name: "Generate changelog"

changelogs/internal/newsfragments/2088.clarification

@@ -0,0 +1 @@
+Replace Hugo shortcodes in OpenAPI output.

changelogs/server_server/newsfragments/2067.clarification

@@ -0,0 +1 @@
+Add a note to the invite endpoints that invites to local users may be received twice over federation if the homeserver is already in the room.

data/api/server-server/invites-v1.yaml

@@ -20,7 +20,7 @@ paths:
     put:
       summary: Invites a remote user to a room
       description: |-
-        Invites a remote user to a room. Once the event has been  signed by both the inviting
+        Invites a remote user to a room. Once the event has been signed by both the inviting
         homeserver and the invited homeserver, it can be sent to all of the servers in the
         room by the inviting homeserver.
 
@@ -32,6 +32,10 @@ paths:
         [room version specification](/rooms) for precise event formats. **The request and response
         bodies here describe the common event fields in more detail and may be missing other
         required fields for a PDU.**
+
+        Also note that if the remote homeserver is already in the room, it will receive the
+        invite event twice; once through this endpoint, and again through a [federation
+        transaction](/server-server-api/#transactions).
       operationId: sendInviteV1
       security:
         - signedRequest: []

data/api/server-server/invites-v2.yaml

@@ -24,7 +24,7 @@ paths:
         This API is nearly identical to the v1 API with the exception of the request
         body being different, and the response format fixed.
 
-        Invites a remote user to a room. Once the event has been  signed by both the inviting
+        Invites a remote user to a room. Once the event has been signed by both the inviting
         homeserver and the invited homeserver, it can be sent to all of the servers in the
         room by the inviting homeserver.
 
@@ -36,6 +36,10 @@ paths:
         [room version specification](/rooms) for precise event formats. **The request and response
         bodies here describe the common event fields in more detail and may be missing other
         required fields for a PDU.**
+
+        Also note that if the remote homeserver is already in the room, it will receive the
+        invite event twice; once through this endpoint, and again through a [federation
+        transaction](/server-server-api/#transactions).
       operationId: sendInviteV2
       security:
         - signedRequest: []

scripts/dump-openapi.py

@@ -32,6 +32,35 @@ import yaml
 scripts_dir = os.path.dirname(os.path.abspath(__file__))
 api_dir = os.path.join(os.path.dirname(scripts_dir), "data", "api")
 
+# Finds a Hugo shortcode in a string.
+#
+# A shortcode is defined as (newlines and whitespaces for presentation purpose):
+#
+# {{%
+#     <zero or more whitespaces>
+#     <name of shortcode>
+#     (optional <one or more whitespaces><list of parameters>)
+#     <zero or more whitespaces>
+# %}}
+#
+# With:
+#
+# * <name of shortcode>: any word character and `-` and `/`. `re.ASCII` is used to only match
+#   ASCII characters in the name.
+# * <list of parameters>: any character except `}`, must not start or end with a
+#   whitespace.
+shortcode_regex = re.compile(r"""\{\{\%                                   # {{%
+                                 \s*                                      # zero or more whitespaces
+                                 (?P<name>[\w/-]+)                        # name of shortcode
+                                 (?:\s+(?P<params>[^\s\}][^\}]+[^\s\}]))? # optional list of parameters
+                                 \s*                                      # zero or more whitespaces
+                                 \%\}\}                                   # %}}""", re.ASCII | re.VERBOSE)
+
+# Parses the parameters of a Hugo shortcode.
+#
+# For simplicity, this currently only supports the `key="value"` format.
+shortcode_params_regex = re.compile(r"(?P<key>\w+)=\"(?P<value>[^\"]+)\"", re.ASCII)
+
 def prefix_absolute_path_references(text, base_url):
     """Adds base_url to absolute-path references.
 
@@ -44,17 +73,90 @@ def prefix_absolute_path_references(text, base_url):
     """
     return text.replace("](/", "]({}/".format(base_url))
 
-def edit_links(node, base_url):
-    """Finds description nodes and makes any links in them absolute."""
+def replace_match(match, replacement):
+    """Replaces the regex match by the replacement in the text."""
+    return match.string[:match.start()] + replacement + match.string[match.end():]
+
+def replace_shortcode(shortcode):
+    """Replaces the shortcode by a Markdown fallback in the text.
+
+    The supported shortcodes are:
+
+    * boxes/note, boxes/rationale, boxes/warning
+    * added-in, changed-in
+    
+    All closing tags (`{{ /shortcode }}`) are replaced with the empty string.
+    """
+
+    if shortcode['name'].startswith("/"):
+        # This is the end of the shortcode, just remove it.
+        return replace_match(shortcode, "")
+
+    # Parse the parameters of the shortcode
+    params = {}
+    if shortcode['params']:
+        for param in shortcode_params_regex.finditer(shortcode['params']):
+            if param['key']:
+                params[param['key']] = param['value']
+
+    match shortcode['name']:
+        case "boxes/note":
+            return replace_match(shortcode, "**NOTE:** ")
+        case "boxes/rationale":
+            return replace_match(shortcode, "**RATIONALE:** ")
+        case "boxes/warning":
+            return replace_match(shortcode, "**WARNING:** ")
+        case "added-in":
+            version = params['v']
+            if not version:
+                raise ValueError("Missing parameter `v` for `added-in` shortcode")
+
+            return replace_match(shortcode, f"**[Added in `v{version}`]** ")
+        case "changed-in":
+            version = params['v']
+            if not version:
+                raise ValueError("Missing parameter `v` for `changed-in` shortcode")
+
+            return replace_match(shortcode, f"**[Changed in `v{version}`]** ")
+        case _:
+            raise ValueError("Unknown shortcode", shortcode['name'])
+
+
+def find_and_replace_shortcodes(text):
+    """Finds Hugo shortcodes and replaces them by a Markdown fallback.
+
+    The supported shortcodes are:
+
+    * boxes/note, boxes/rationale, boxes/warning
+    * added-in, changed-in
+    """
+    # We use a `while` loop with `search` instead of a `for` loop with
+    # `finditer`, because as soon as we start replacing text, the
+    # indices of the match are invalid.
+    while shortcode := shortcode_regex.search(text):
+        text = replace_shortcode(shortcode)
+
+    return text
+
+def edit_descriptions(node, base_url):
+    """Finds description nodes and apply fixes to them.
+
+    The fixes that are applied are:
+
+    * Make links absolute
+    * Replace Hugo shortcodes
+    """
     if isinstance(node, dict):
         for key in node:
             if isinstance(node[key], str):
                 node[key] = prefix_absolute_path_references(node[key], base_url)
+                node[key] = find_and_replace_shortcodes(node[key])
             else:
-                edit_links(node[key], base_url)
+                edit_descriptions(node[key], base_url)
     elif isinstance(node, list):
         for item in node:
-            edit_links(item, base_url)
+            edit_descriptions(item, base_url)
+
 
 parser = argparse.ArgumentParser(
     "dump-openapi.py - assemble the OpenAPI specs into a single JSON file"
@@ -164,7 +266,7 @@ for filename in os.listdir(selected_api_dir):
 if untagged != 0:
     print("{} untagged operations, you may want to look into fixing that.".format(untagged))
 
-edit_links(output, base_url)
+edit_descriptions(output, base_url)
 
 print("Generating %s" % output_file)