From 41de0b1bf752c77098f0ff8ed30a360f10502dee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Commaille?= Date: Tue, 25 Feb 2025 18:27:06 +0100 Subject: [PATCH 1/7] Replace Hugo shortcodes in OpenAPI output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Kévin Commaille --- scripts/dump-openapi.py | 111 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 106 insertions(+), 5 deletions(-) diff --git a/scripts/dump-openapi.py b/scripts/dump-openapi.py index 490ac9bf..f459ef9b 100755 --- a/scripts/dump-openapi.py +++ b/scripts/dump-openapi.py @@ -32,6 +32,29 @@ import yaml scripts_dir = os.path.dirname(os.path.abspath(__file__)) api_dir = os.path.join(os.path.dirname(scripts_dir), "data", "api") +# Finds a Hugo shortcode in a string. +# +# A shortcode is defined as (newlines and whitespaces for presentation purpose): +# +# {{% +# +# +# +# (optional ) +# %}} +# +# With: +# +# * : any word character and `-` and `/`. +# * : any character except `}`, must not start or end with a +# whitespace. +shortcode_regex = re.compile(r"\{\{\%\s+(?P[\w\/-]+)\s+(?:(?P[^\s\}][^\}]+[^\s\}])\s+)?\%\}\}", re.ASCII) + +# Parses the parameters of a Hugo shortcode. +# +# For simplicity, this currently only supports the `key="value"` format. +shortcode_params_regex = re.compile(r"(?P\w+)=\"(?P[^\"]+)\"", re.ASCII) + def prefix_absolute_path_references(text, base_url): """Adds base_url to absolute-path references. @@ -44,17 +67,95 @@ def prefix_absolute_path_references(text, base_url): """ return text.replace("](/", "]({}/".format(base_url)) -def edit_links(node, base_url): - """Finds description nodes and makes any links in them absolute.""" +def replace_match(text, match, replacement): + """Replaces the regex match by the replacement in the text.""" + return text[:match.start()] + replacement + text[match.end():] + +def replace_shortcode(text, shortcode): + """Replaces the shortcode by a Markdown fallback in the text. + + The supported shortcodes are: + + * boxes/note, boxes/rationale, boxes/warning + * added-in, changed-in + """ + + if shortcode['name'].startswith("/"): + # This is the end of the shortcode, just remove it. + return replace_match(text, shortcode['match'], "") + + match shortcode['name']: + case "boxes/note": + text = replace_match(text, shortcode['match'], "**NOTE:** ") + case "boxes/rationale": + text = replace_match(text, shortcode['match'], "**RATIONALE:** ") + case "boxes/warning": + text = replace_match(text, shortcode['match'], "**WARNING:** ") + case "added-in": + version = shortcode['params']['v'] + if not version: + raise ValueError("Missing parameter `v` for `added-in` shortcode") + + text = replace_match(text, shortcode['match'], f"**[Added in `v{version}`]** ") + case "changed-in": + version = shortcode['params']['v'] + if not version: + raise ValueError("Missing parameter `v` for `changed-in` shortcode") + + text = replace_match(text, shortcode['match'], f"**[Changed in `v{version}`]** ") + case _: + raise ValueError("Unknown shortcode", shortcode['name']) + + return text + + +def find_and_replace_shortcodes(text): + """Finds Hugo shortcodes and replaces them by a Markdown fallback. + + The supported shortcodes are: + + * boxes/note, boxes/rationale, boxes/warning + * added-in, changed-in + """ + # We use a `while` loop with `search` instead of a `for` loop with + # `finditer`, because as soon as we start replacing text, the + # indices of the match are invalid. + while match := shortcode_regex.search(text): + # Parse the parameters of the shortcode + params = {} + if match['params']: + for param in shortcode_params_regex.finditer(match['params']): + if param['key']: + params[param['key']] = param['value'] + + shortcode = { + 'name': match['name'], + 'params': params, + 'match': match, + } + text = replace_shortcode(text, shortcode) + + return text + +def edit_descriptions(node, base_url): + """Finds description nodes and apply fixes to them. + + The fixes that are applied are: + + * Make links absolute + * Replace shortcodes + """ if isinstance(node, dict): for key in node: if isinstance(node[key], str): node[key] = prefix_absolute_path_references(node[key], base_url) + node[key] = find_and_replace_shortcodes(node[key]) else: - edit_links(node[key], base_url) + edit_descriptions(node[key], base_url) elif isinstance(node, list): for item in node: - edit_links(item, base_url) + edit_descriptions(item, base_url) + parser = argparse.ArgumentParser( "dump-openapi.py - assemble the OpenAPI specs into a single JSON file" @@ -164,7 +265,7 @@ for filename in os.listdir(selected_api_dir): if untagged != 0: print("{} untagged operations, you may want to look into fixing that.".format(untagged)) -edit_links(output, base_url) +edit_descriptions(output, base_url) print("Generating %s" % output_file) From 8602a1ae8f86680bf80c780b05a01a4834cf0554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Commaille?= Date: Tue, 25 Feb 2025 18:31:12 +0100 Subject: [PATCH 2/7] Add changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Kévin Commaille --- changelogs/internal/newsfragments/2088.clarification | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/internal/newsfragments/2088.clarification diff --git a/changelogs/internal/newsfragments/2088.clarification b/changelogs/internal/newsfragments/2088.clarification new file mode 100644 index 00000000..a0e53726 --- /dev/null +++ b/changelogs/internal/newsfragments/2088.clarification @@ -0,0 +1 @@ +Replace Hugo shortcodes in OpenAPI output. From 73b1936daeffe681b7294bff67138370e8d65904 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Commaille?= Date: Tue, 25 Feb 2025 18:35:51 +0100 Subject: [PATCH 3/7] Upgrade Python in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Kévin Commaille --- .github/workflows/main.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 25a2fb68..daa8fb67 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,6 +2,7 @@ name: "Spec" env: HUGO_VERSION: 0.139.0 + PYTHON_VERSION: 3.13 on: push: @@ -40,7 +41,7 @@ jobs: - name: "➕ Setup Python" uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' cache-dependency-path: scripts/requirements.txt - name: "➕ Install dependencies" @@ -59,7 +60,7 @@ jobs: - name: "➕ Setup Python" uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' cache-dependency-path: scripts/requirements.txt - name: "➕ Install dependencies" @@ -78,7 +79,7 @@ jobs: - name: "➕ Setup Python" uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' cache-dependency-path: scripts/requirements.txt - name: "➕ Install dependencies" @@ -120,7 +121,7 @@ jobs: - name: "➕ Setup Python" uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' cache-dependency-path: scripts/requirements.txt - name: "➕ Install dependencies" @@ -172,7 +173,7 @@ jobs: - name: "➕ Setup Python" uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: ${{ env.PYTHON_VERSION }} - name: "➕ Install towncrier" run: "pip install 'towncrier'" - name: "Generate changelog" From 69adb6b2709e01ce2170510057d2dcffb3ee8d42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Commaille?= Date: Tue, 6 May 2025 10:52:47 +0200 Subject: [PATCH 4/7] Remove trailing whitespaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Kévin Commaille --- scripts/dump-openapi.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/dump-openapi.py b/scripts/dump-openapi.py index f459ef9b..79a07e73 100755 --- a/scripts/dump-openapi.py +++ b/scripts/dump-openapi.py @@ -50,7 +50,7 @@ api_dir = os.path.join(os.path.dirname(scripts_dir), "data", "api") # whitespace. shortcode_regex = re.compile(r"\{\{\%\s+(?P[\w\/-]+)\s+(?:(?P[^\s\}][^\}]+[^\s\}])\s+)?\%\}\}", re.ASCII) -# Parses the parameters of a Hugo shortcode. +# Parses the parameters of a Hugo shortcode. # # For simplicity, this currently only supports the `key="value"` format. shortcode_params_regex = re.compile(r"(?P\w+)=\"(?P[^\"]+)\"", re.ASCII) @@ -73,7 +73,7 @@ def replace_match(text, match, replacement): def replace_shortcode(text, shortcode): """Replaces the shortcode by a Markdown fallback in the text. - + The supported shortcodes are: * boxes/note, boxes/rationale, boxes/warning @@ -83,7 +83,7 @@ def replace_shortcode(text, shortcode): if shortcode['name'].startswith("/"): # This is the end of the shortcode, just remove it. return replace_match(text, shortcode['match'], "") - + match shortcode['name']: case "boxes/note": text = replace_match(text, shortcode['match'], "**NOTE:** ") @@ -95,13 +95,13 @@ def replace_shortcode(text, shortcode): version = shortcode['params']['v'] if not version: raise ValueError("Missing parameter `v` for `added-in` shortcode") - + text = replace_match(text, shortcode['match'], f"**[Added in `v{version}`]** ") case "changed-in": version = shortcode['params']['v'] if not version: raise ValueError("Missing parameter `v` for `changed-in` shortcode") - + text = replace_match(text, shortcode['match'], f"**[Changed in `v{version}`]** ") case _: raise ValueError("Unknown shortcode", shortcode['name']) @@ -111,7 +111,7 @@ def replace_shortcode(text, shortcode): def find_and_replace_shortcodes(text): """Finds Hugo shortcodes and replaces them by a Markdown fallback. - + The supported shortcodes are: * boxes/note, boxes/rationale, boxes/warning @@ -127,7 +127,7 @@ def find_and_replace_shortcodes(text): for param in shortcode_params_regex.finditer(match['params']): if param['key']: params[param['key']] = param['value'] - + shortcode = { 'name': match['name'], 'params': params, @@ -139,7 +139,7 @@ def find_and_replace_shortcodes(text): def edit_descriptions(node, base_url): """Finds description nodes and apply fixes to them. - + The fixes that are applied are: * Make links absolute From 163bfe7c2c0a7a0fac6eaffb86f2f2e24de68816 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Commaille?= Date: Tue, 6 May 2025 10:53:49 +0200 Subject: [PATCH 5/7] Improve doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Kévin Commaille --- scripts/dump-openapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dump-openapi.py b/scripts/dump-openapi.py index 79a07e73..0df4ce92 100755 --- a/scripts/dump-openapi.py +++ b/scripts/dump-openapi.py @@ -143,7 +143,7 @@ def edit_descriptions(node, base_url): The fixes that are applied are: * Make links absolute - * Replace shortcodes + * Replace Hugo shortcodes """ if isinstance(node, dict): for key in node: From c1c90a6b3bdb5ce916f3f36ee7fd73f656089ae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Commaille?= Date: Tue, 6 May 2025 11:20:56 +0200 Subject: [PATCH 6/7] Fix and split regex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Kévin Commaille --- scripts/dump-openapi.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/dump-openapi.py b/scripts/dump-openapi.py index 0df4ce92..3b57d5de 100755 --- a/scripts/dump-openapi.py +++ b/scripts/dump-openapi.py @@ -37,10 +37,10 @@ api_dir = os.path.join(os.path.dirname(scripts_dir), "data", "api") # A shortcode is defined as (newlines and whitespaces for presentation purpose): # # {{% -# +# # -# -# (optional ) +# (optional ) +# # %}} # # With: @@ -48,7 +48,12 @@ api_dir = os.path.join(os.path.dirname(scripts_dir), "data", "api") # * : any word character and `-` and `/`. # * : any character except `}`, must not start or end with a # whitespace. -shortcode_regex = re.compile(r"\{\{\%\s+(?P[\w\/-]+)\s+(?:(?P[^\s\}][^\}]+[^\s\}])\s+)?\%\}\}", re.ASCII) +shortcode_regex = re.compile(r"""\{\{\% # {{% + \s* # zero or more whitespaces + (?P[\w/-]+) # name of shortcode + (?:\s+(?P[^\s\}][^\}]+[^\s\}]))? # optional list of parameters + \s* # zero or more whitespaces + \%\}\} # %}}""", re.ASCII | re.VERBOSE) # Parses the parameters of a Hugo shortcode. # From a23985296efdae2c62928bbe3706e97b85abd820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Commaille?= Date: Tue, 6 May 2025 12:09:28 +0200 Subject: [PATCH 7/7] Simplify code to replace shortcodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Kévin Commaille --- scripts/dump-openapi.py | 47 ++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/scripts/dump-openapi.py b/scripts/dump-openapi.py index 3b57d5de..c064c2bd 100755 --- a/scripts/dump-openapi.py +++ b/scripts/dump-openapi.py @@ -72,11 +72,11 @@ def prefix_absolute_path_references(text, base_url): """ return text.replace("](/", "]({}/".format(base_url)) -def replace_match(text, match, replacement): +def replace_match(match, replacement): """Replaces the regex match by the replacement in the text.""" - return text[:match.start()] + replacement + text[match.end():] + return match.string[:match.start()] + replacement + match.string[match.end():] -def replace_shortcode(text, shortcode): +def replace_shortcode(shortcode): """Replaces the shortcode by a Markdown fallback in the text. The supported shortcodes are: @@ -87,32 +87,37 @@ def replace_shortcode(text, shortcode): if shortcode['name'].startswith("/"): # This is the end of the shortcode, just remove it. - return replace_match(text, shortcode['match'], "") + return replace_match(shortcode, "") + + # Parse the parameters of the shortcode + params = {} + if shortcode['params']: + for param in shortcode_params_regex.finditer(shortcode['params']): + if param['key']: + params[param['key']] = param['value'] match shortcode['name']: case "boxes/note": - text = replace_match(text, shortcode['match'], "**NOTE:** ") + return replace_match(shortcode, "**NOTE:** ") case "boxes/rationale": - text = replace_match(text, shortcode['match'], "**RATIONALE:** ") + return replace_match(shortcode, "**RATIONALE:** ") case "boxes/warning": - text = replace_match(text, shortcode['match'], "**WARNING:** ") + return replace_match(shortcode, "**WARNING:** ") case "added-in": - version = shortcode['params']['v'] + version = params['v'] if not version: raise ValueError("Missing parameter `v` for `added-in` shortcode") - text = replace_match(text, shortcode['match'], f"**[Added in `v{version}`]** ") + return replace_match(shortcode, f"**[Added in `v{version}`]** ") case "changed-in": - version = shortcode['params']['v'] + version = params['v'] if not version: raise ValueError("Missing parameter `v` for `changed-in` shortcode") - text = replace_match(text, shortcode['match'], f"**[Changed in `v{version}`]** ") + return replace_match(shortcode, f"**[Changed in `v{version}`]** ") case _: raise ValueError("Unknown shortcode", shortcode['name']) - return text - def find_and_replace_shortcodes(text): """Finds Hugo shortcodes and replaces them by a Markdown fallback. @@ -125,20 +130,8 @@ def find_and_replace_shortcodes(text): # We use a `while` loop with `search` instead of a `for` loop with # `finditer`, because as soon as we start replacing text, the # indices of the match are invalid. - while match := shortcode_regex.search(text): - # Parse the parameters of the shortcode - params = {} - if match['params']: - for param in shortcode_params_regex.finditer(match['params']): - if param['key']: - params[param['key']] = param['value'] - - shortcode = { - 'name': match['name'], - 'params': params, - 'match': match, - } - text = replace_shortcode(text, shortcode) + while shortcode := shortcode_regex.search(text): + text = replace_shortcode(shortcode) return text