chore: introduce code snippet linting infra (#23960)

.github/workflows/infra.yml

@@ -37,3 +37,20 @@ jobs:
         fi
     - name: Audit prod NPM dependencies
       run: npm audit --omit dev
+  lint-snippets:
+    name: "Lint snippets"
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 18
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+    - uses: actions/setup-dotnet@v3
+      with:
+        dotnet-version: 8.0.x
+    - run: npm ci
+    - run: pip install -r utils/doclint/linting-code-snippets/python/requirements.txt
+    - run: node utils/doclint/linting-code-snippets/cli.js

package-lock.json

@@ -12,6 +12,7 @@
         "packages/*"
       ],
       "devDependencies": {
+        "@actions/core": "^1.10.0",
         "@babel/cli": "^7.19.3",
         "@babel/plugin-proposal-class-properties": "^7.18.6",
         "@babel/plugin-proposal-export-namespace-from": "^7.18.9",
@@ -78,6 +79,25 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/@actions/core": {
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.0.tgz",
+      "integrity": "sha512-2aZDDa3zrrZbP5ZYg159sNoLRb61nQ7awl5pSvIq5Qpj81vwDzdMRKzkWJGJuwVvWpvZKx7vspJALyvaaIQyug==",
+      "dev": true,
+      "dependencies": {
+        "@actions/http-client": "^2.0.1",
+        "uuid": "^8.3.2"
+      }
+    },
+    "node_modules/@actions/http-client": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.1.0.tgz",
+      "integrity": "sha512-BonhODnXr3amchh4qkmjPMUO8mFi/zLaaCeCAJZqch8iQqyDnVIkySjB38VHAC8IJ+bnlgfOqlhpyCUZHlQsqw==",
+      "dev": true,
+      "dependencies": {
+        "tunnel": "^0.0.6"
+      }
+    },
     "node_modules/@ampproject/remapping": {
       "version": "2.2.0",
       "license": "Apache-2.0",
@@ -5785,7 +5805,6 @@
       "version": "0.0.6",
       "dev": true,
       "license": "MIT",
-      "optional": true,
       "engines": {
         "node": ">=0.6.11 <=0.7.0 || >=0.7.3"
       }
@@ -5886,6 +5905,15 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/uuid": {
+      "version": "8.3.2",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
+      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
+      "dev": true,
+      "bin": {
+        "uuid": "dist/bin/uuid"
+      }
+    },
     "node_modules/validate-html-nesting": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/validate-html-nesting/-/validate-html-nesting-1.2.2.tgz",
@@ -6560,6 +6588,25 @@
       "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==",
       "dev": true
     },
+    "@actions/core": {
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.0.tgz",
+      "integrity": "sha512-2aZDDa3zrrZbP5ZYg159sNoLRb61nQ7awl5pSvIq5Qpj81vwDzdMRKzkWJGJuwVvWpvZKx7vspJALyvaaIQyug==",
+      "dev": true,
+      "requires": {
+        "@actions/http-client": "^2.0.1",
+        "uuid": "^8.3.2"
+      }
+    },
+    "@actions/http-client": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.1.0.tgz",
+      "integrity": "sha512-BonhODnXr3amchh4qkmjPMUO8mFi/zLaaCeCAJZqch8iQqyDnVIkySjB38VHAC8IJ+bnlgfOqlhpyCUZHlQsqw==",
+      "dev": true,
+      "requires": {
+        "tunnel": "^0.0.6"
+      }
+    },
     "@ampproject/remapping": {
       "version": "2.2.0",
       "requires": {
@@ -10357,8 +10404,7 @@
     },
     "tunnel": {
       "version": "0.0.6",
-      "dev": true,
-      "optional": true
+      "dev": true
     },
     "type-check": {
       "version": "0.4.0",
@@ -10414,6 +10460,12 @@
       "version": "1.0.3",
       "dev": true
     },
+    "uuid": {
+      "version": "8.3.2",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
+      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
+      "dev": true
+    },
     "validate-html-nesting": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/validate-html-nesting/-/validate-html-nesting-1.2.2.tgz",

package.json

@@ -47,6 +47,7 @@
     "packages/*"
   ],
   "devDependencies": {
+    "@actions/core": "^1.10.0",
     "@babel/cli": "^7.19.3",
     "@babel/plugin-proposal-class-properties": "^7.18.6",
     "@babel/plugin-proposal-export-namespace-from": "^7.18.9",

utils/doclint/linting-code-snippets/cli.js

@@ -0,0 +1,292 @@
+#!/usr/bin/env node
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+const debug = require('debug')
+const fs = require('fs');
+const path = require('path');
+const { parseApi } = require('../api_parser');
+const md = require('../../markdown');
+const { ESLint } = require('eslint')
+const child_process = require('child_process');
+const os = require('os');
+const actions = require('@actions/core')
+
+/** @typedef {import('../documentation').Type} Type */
+/** @typedef {import('../../markdown').MarkdownNode} MarkdownNode */
+
+const PROJECT_DIR = path.join(__dirname, '..', '..', '..');
+
+function getAllMarkdownFiles(dirPath, filePaths = []) {
+  for (const entry of fs.readdirSync(dirPath, { withFileTypes: true })) {
+    if (entry.isFile() && entry.name.toLowerCase().endsWith('.md'))
+      filePaths.push(path.join(dirPath, entry.name));
+    else if (entry.isDirectory())
+      getAllMarkdownFiles(path.join(dirPath, entry.name), filePaths);
+  }
+  return filePaths;
+}
+
+const run = async () => {
+  const documentationRoot = path.join(PROJECT_DIR, 'docs', 'src');
+  const lintingServiceFactory = new LintingServiceFactory();
+  let documentation = parseApi(path.join(documentationRoot, 'api'));
+
+  /** @type {CodeSnippet[]} */
+  const codeSnippets = [];
+  for (const filePath of getAllMarkdownFiles(documentationRoot)) {
+    const data = fs.readFileSync(filePath, 'utf-8');
+    let rootNode = md.parse(data);
+    // Renders links.
+    documentation.renderLinksInNodes(rootNode);
+    documentation.generateSourceCodeComments();
+    md.visitAll(rootNode, node => {
+      if (node.type !== 'code')
+        return;
+      const codeLang = node.codeLang.split(' ')[0];
+      const code = node.lines.join('\n');
+      codeSnippets.push({
+        filePath,
+        codeLang,
+        code,
+      })
+    });
+  }
+  await lintingServiceFactory.lintAndReport(codeSnippets);
+  const { hasErrors } = lintingServiceFactory.reportMetrics();
+  if (hasErrors)
+    process.exit(1);
+}
+
+
+/** @typedef {{ codeLang: string, code: string, filePath: string }} CodeSnippet */
+/** @typedef {{ status: 'ok' | 'updated' | 'error' | 'unsupported', error?: string }} LintResult */
+
+class LintingService {
+  /**
+   * @param {string} codeLang
+   * @returns {boolean}
+   */
+  supports(codeLang) {
+    throw new Error('supports() is not implemented');
+  }
+
+  async _writeTempSnippetsFile(snippets) {
+    const tempFile = path.join(os.tmpdir(), `snippet-${Date.now()}.json`);
+    await fs.promises.writeFile(tempFile, JSON.stringify(snippets, undefined, 2));
+    return tempFile;
+  }
+
+  /**
+   * @param {string} command 
+   * @param {string[]} args
+   * @param {CodeSnippet[]} snippets 
+   * @param {string} cwd
+   * @returns {Promise<LintResult[]>}
+   */
+  async spawnAsync(command, args, snippets, cwd) {
+    const tempFile = await this._writeTempSnippetsFile(snippets);
+    return await new Promise((fulfill, reject) => {
+      const child = child_process.spawn(command, [...args, tempFile], { cwd });
+      let stdout = '';
+      child.on('error', reject);
+      child.stdout.on('data', data => stdout += data.toString());
+      child.stderr.pipe(process.stderr);
+      child.on('exit', code => {
+        if (code)
+          reject(new Error(`${command} exited with code ${code}`));
+        else
+          fulfill(JSON.parse(stdout));
+      });
+    });
+  }
+
+  /**
+   * @param {CodeSnippet[]} snippets 
+   * @returns {Promise<LintResult[]>}
+   */
+  async lint(snippets) {
+    throw new Error('lint() is not implemented');
+  }
+}
+
+
+class JSLintingService extends LintingService {
+  _knownBadSnippets = [
+    'mount(',
+    'render(',
+    'vue-router',
+    'experimental-ct',
+  ];
+  constructor() {
+    super();
+    this.eslint = new ESLint({
+      overrideConfigFile: path.join(PROJECT_DIR, '.eslintrc.js'),
+      useEslintrc: false,
+      overrideConfig: {
+        rules: {
+          'notice/notice': 'off',
+          '@typescript-eslint/no-unused-vars': 'off',
+        },
+      }
+    });
+  }
+
+  supports(codeLang) {
+    return codeLang === 'js' || codeLang === 'ts';
+  }
+
+  /**
+   * @param {CodeSnippet} snippet
+   * @returns {Promise<LintResult>}
+   */
+  async _lintSnippet(snippet) {
+    if (this._knownBadSnippets.some(s => snippet.code.includes(s)))
+      return { status: 'ok' };
+    const results = await this.eslint.lintText(snippet.code);
+    if (!results || !results.length || !results[0].messages.length)
+      return { status: 'ok' };
+    return { status: 'error', error: results[0].messages[0].message };
+  }
+
+  /**
+   * @param {CodeSnippet[]} snippets
+   * @returns {Promise<LintResult[]>}
+   */
+  async lint(snippets) {
+    return Promise.all(snippets.map(async snippet => this._lintSnippet(snippet)));
+  }
+}
+
+class PythonLintingService extends LintingService {
+  supports(codeLang) {
+    return codeLang === 'python' || codeLang === 'py';
+  }
+
+  async lint(snippets) {
+    const result = await this.spawnAsync('python', [path.join(__dirname, 'python', 'main.py')], snippets, path.join(__dirname, 'python'))
+    return result;
+  }
+}
+
+class CSharpLintingService extends LintingService {
+  supports(codeLang) {
+    return codeLang === 'csharp';
+  }
+
+  async lint(snippets) {
+    return await this.spawnAsync('dotnet', ['run', '--project', path.join(__dirname, 'csharp')], snippets, path.join(__dirname, 'csharp'))
+  }
+}
+
+class LintingServiceFactory {
+  constructor() {
+    /** @type {LintingService[]} */
+    this.services = [
+      new JSLintingService(),
+    ]
+    if (!process.env.NO_EXTERNAL_DEPS) {
+      this.services.push(
+        new PythonLintingService(),
+        new CSharpLintingService(),
+      );
+    }
+    this._metrics = {};
+    this._log = debug('linting-service');
+  }
+
+  /**
+   * @param {CodeSnippet[]} allSnippets
+   */
+  async lintAndReport(allSnippets) {
+    /** @type {Record<string, CodeSnippet[]>} */
+    const groupedByLanguage = allSnippets.reduce((acc, snippet) => {
+      if (!acc[snippet.codeLang])
+        acc[snippet.codeLang] = [];
+      acc[snippet.codeLang].push(snippet);
+      return acc;
+    }, {});
+    for (const language in groupedByLanguage) {
+      const service = this.services.find(service => service.supports(language));
+      if (!service) {
+        this._collectMetrics(language, {
+          status: 'unsupported',
+        });
+        continue;
+      }
+      const languageSnippets = groupedByLanguage[language];
+      const results = await service.lint(languageSnippets);
+      if (results.length !== languageSnippets.length)
+        throw new Error('Linting service returned wrong number of results');
+
+      for (const [{ code, codeLang, filePath }, result] of /** @type {[[CodeSnippet, LintResult]]} */ (results.map((result, index) => [languageSnippets[index], result]))) {
+        const { status, error } = result;
+        this._collectMetrics(codeLang, result);
+        if (status === 'error') {
+          console.log(`${codeLang} linting error!`);
+          console.log(`ERROR: ${error}`);
+          console.log(`File: ${filePath}`);
+          console.log(code);
+          console.log('-'.repeat(80));
+          if (process.env.GITHUB_ACTION)
+            actions.warning(`Error: ${error}\nUnable to lint:\n${code}`, {
+              title: `${codeLang} linting error`,
+              file: filePath,
+            });
+        }
+      }
+    }
+  }
+
+  /**
+   * @returns {{ hasErrors: boolean }}
+   */
+  reportMetrics() {
+    console.log('Metrics:');
+    const renderMetric = (metric, name) => {
+      if (!metric[name])
+        return '';
+      return `${name}: ${metric[name]}`;
+    }
+    let hasErrors = false;
+    const languagesOrderedByOk = Object.entries(this._metrics).sort(([langA], [langB]) => {
+      return this._metrics[langB].ok - this._metrics[langA].ok
+    })
+    for (const [language, metrics] of languagesOrderedByOk) {
+      if (metrics.error)
+        hasErrors = true;
+      console.log(`  ${language}: ${['ok', 'updated', 'error', 'unsupported'].map(name => renderMetric(metrics, name)).filter(Boolean).join(', ')}`)
+    }
+    return { hasErrors }
+  }
+
+  /**
+   * @param {string} language 
+   * @param {LintResult} result 
+   */
+  _collectMetrics(language, result) {
+    if (!this._metrics[language])
+      this._metrics[language] = { ok: 0, updated: 0, error: 0, unsupported: 0 };
+    this._metrics[language][result.status]++;
+  }
+}
+
+run().catch(e => {
+  console.error(e);
+  process.exit(1);
+});

utils/doclint/linting-code-snippets/csharp/.gitignore

@@ -0,0 +1,2 @@
+obj/
+bin/

utils/doclint/linting-code-snippets/csharp/Program.cs

@@ -0,0 +1,43 @@
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Text.Json;
+using System.Threading.Tasks;
+
+var codeSnippetsPath = args[args.Length - 1];
+var codeSnippets = JsonSerializer.Deserialize<List<CodeSnippet>>(File.ReadAllText(codeSnippetsPath));
+if (codeSnippets == null)
+{
+    Console.WriteLine("Error: codeSnippets is null");
+    return;
+}
+var output = new List<object>();
+
+foreach (var codeSnippet in codeSnippets)
+{
+    var tree = CSharpSyntaxTree.ParseText(codeSnippet.code);
+    var syntaxErrors = tree.GetDiagnostics()
+        .Where(diag => diag.Severity == DiagnosticSeverity.Error)
+        .ToList();
+    if (syntaxErrors.Any())
+    {
+        output.Add(new
+        {
+            status = "error",
+            error = string.Join("\n", syntaxErrors.Select(diag => diag.GetMessage()))
+        });
+    }
+    else
+    {
+        output.Add(new
+        {
+            status = "ok"
+        });
+    }
+}
+
+Console.WriteLine(JsonSerializer.Serialize(output));
+
+record CodeSnippet(string filePath, string codeLang, string code);

utils/doclint/linting-code-snippets/csharp/linting-code-snippets.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <RootNamespace>csharp</RootNamespace>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.CodeAnalysis" Version="4.6.0" />
+  </ItemGroup>
+
+</Project>

utils/doclint/linting-code-snippets/python/main.py

@@ -0,0 +1,34 @@
+import json
+import sys
+import black
+
+def check_code_snippet(code_snippet: str):
+    try:
+        formatted_code = black.format_str(code_snippet, mode=black.FileMode())
+    except Exception as e:
+        return {
+            'status': 'error',
+            'error': str(e),
+        }
+    if formatted_code.strip() == code_snippet.strip():
+        return {
+            'status': 'success',
+        }
+    return {
+        'status': 'updated',
+        'newCode': formatted_code,
+    }
+
+
+def main():
+    code_snippets_path = sys.argv[1]
+    if not code_snippets_path:
+        print("No code snippets path provided")
+        return
+    code_snippets = json.load(open(code_snippets_path))
+    formatted_codes = [check_code_snippet(snippet["code"]) for snippet in code_snippets]
+    print(json.dumps(formatted_codes))
+
+
+if __name__ == "__main__":
+    main()

utils/doclint/linting-code-snippets/python/requirements.txt

@@ -0,0 +1 @@
+black==23.3.0