docs: reformat api-body to allow multiline params documentation (#4604)

docs-src/api-params.md

@@ -309,7 +309,7 @@ Network proxy settings to use with this context. Note that browser needs to be l
 option to work. If all contexts override the proxy, global proxy will be never used and can be any string, for example
 `launch({ proxy: { server: 'per-context' } })`.
 
-## shared-context-params
+## shared-context-params-list
 - %%-context-option-acceptdownloads-%%
 - %%-context-option-ignorehttpserrors-%%
 - %%-context-option-bypasscsp-%%

docs/api.md

@@ -1415,7 +1415,7 @@ Shortcut for main frame's [frame.evaluate(pageFunction[, arg])](#frameevaluatepa
 #### page.evaluateHandle(pageFunction[, arg])
 - `pageFunction` <[function]|[string]> Function to be evaluated in the page context
 - `arg` <[EvaluationArgument]> Optional argument to pass to `pageFunction`
-- returns: <[Promise]<[JSHandle]>> Promise which resolves to the return value of `pageFunction` as in-page object (JSHandle)
+- returns: <[Promise]<[JSHandle]>> Promise which resolves to the return value of `pageFunction` as in-page object (JSHandle).
 
 The only difference between `page.evaluate` and `page.evaluateHandle` is that `page.evaluateHandle` returns in-page
 object (JSHandle).
@@ -2597,7 +2597,7 @@ await bodyHandle.dispose();
 #### frame.evaluateHandle(pageFunction[, arg])
 - `pageFunction` <[function]|[string]> Function to be evaluated in the page context
 - `arg` <[EvaluationArgument]> Optional argument to pass to `pageFunction`
-- returns: <[Promise]<[JSHandle]>> Promise which resolves to the return value of `pageFunction` as in-page object (JSHandle)
+- returns: <[Promise]<[JSHandle]>> Promise which resolves to the return value of `pageFunction` as in-page object (JSHandle).
 
 The only difference between `frame.evaluate` and `frame.evaluateHandle` is that `frame.evaluateHandle` returns in-page
 object (JSHandle).
@@ -3616,7 +3616,7 @@ expect(await tweetHandle.evaluate((node, suffix) => node.innerText, ' retweets')
 #### jsHandle.evaluateHandle(pageFunction[, arg])
 - `pageFunction` <[function]|[string]> Function to be evaluated
 - `arg` <[EvaluationArgument]> Optional argument to pass to `pageFunction`
-- returns: <[Promise]<[JSHandle]>> Promise which resolves to the return value of `pageFunction` as in-page object (JSHandle)
+- returns: <[Promise]<[JSHandle]>> Promise which resolves to the return value of `pageFunction` as in-page object (JSHandle).
 
 This method passes this handle as the first argument to `pageFunction`.
 
@@ -4636,7 +4636,7 @@ If the function passed to the `worker.evaluate` returns a non-[Serializable] val
 #### worker.evaluateHandle(pageFunction[, arg])
 - `pageFunction` <[function]|[string]> Function to be evaluated in the page context
 - `arg` <[EvaluationArgument]> Optional argument to pass to `pageFunction`
-- returns: <[Promise]<[JSHandle]>> Promise which resolves to the return value of `pageFunction` as in-page object (JSHandle)
+- returns: <[Promise]<[JSHandle]>> Promise which resolves to the return value of `pageFunction` as in-page object (JSHandle).
 
 The only difference between `worker.evaluate` and `worker.evaluateHandle` is that `worker.evaluateHandle` returns
 in-page object (JSHandle).

utils/doclint/cli.js

@@ -21,7 +21,7 @@ const path = require('path');
 const os = require('os');
 const Source = require('./Source');
 const Message = require('./Message');
-const { renderMdTemplate, parseMd, renderMd, parseArgument } = require('./../parse_md');
+const { parseMd, renderMd, parseArgument } = require('./../parse_md');
 const { spawnSync } = require('child_process');
 const preprocessor = require('./preprocessor');
 
@@ -40,85 +40,148 @@ run().catch(e => {
 async function run() {
   const startTime = Date.now();
 
+  const api = await Source.readFile(path.join(PROJECT_DIR, 'docs', 'api.md'));
+  const readme = await Source.readFile(path.join(PROJECT_DIR, 'README.md'));
+  const binReadme = await Source.readFile(path.join(PROJECT_DIR, 'bin', 'README.md'));
+  const contributing = await Source.readFile(path.join(PROJECT_DIR, 'CONTRIBUTING.md'));
+  const docs = await Source.readdir(path.join(PROJECT_DIR, 'docs'), '.md');
+  const mdSources = [readme, binReadme, api, contributing, ...docs];
+
   /** @type {!Array<!Message>} */
   const messages = [];
   let changedFiles = false;
 
   // Produce api.md
-  const api = await Source.readFile(path.join(PROJECT_DIR, 'docs', 'api.md'));
   {
     const comment = '<!-- THIS FILE IS NOW GENERATED -->';
     const header = fs.readFileSync(path.join(PROJECT_DIR, 'docs-src', 'api-header.md')).toString();
     const body = fs.readFileSync(path.join(PROJECT_DIR, 'docs-src', 'api-body.md')).toString();
     const footer = fs.readFileSync(path.join(PROJECT_DIR, 'docs-src', 'api-footer.md')).toString();
     let params = fs.readFileSync(path.join(PROJECT_DIR, 'docs-src', 'api-params.md')).toString();
-    params = renderMdTemplate(params, params);
 
+    const paramsMap = new Map();
+    for (const node of parseMd(params)) {
+      if (node.h2.endsWith('-list')) {
+        node.children = node.children.map(child => paramsMap.get(child.li));
+        paramsMap.set('%%-' + node.h2 + '-%%', node);
+        continue;
+      }
+      if (node.children[1])
+        node.children[0].li += ' ' + node.children[1].text;
+      paramsMap.set('%%-' + node.h2 + '-%%', node.children[0]);
+    }
+  
     // Generate signatures
     {
-      const nodes = parseMd(renderMdTemplate(body, params));
+      const nodes = parseMd(body);
       const signatures = new Map();
-      let lastMethod;
+      for (const clazz of nodes) {
-      let args;
+        clazz.h3 = clazz.h1;
-      const flushMethodSignature = () => {
+        clazz.h1 = undefined;
-        if (!lastMethod)
+        for (const member of clazz.children) {
-          return;
+          if (!member.h2)
-        const tokens = [];
+            continue;
-        let hasOptional = false;
+          member.h4 = member.h2;
-        for (const arg of args) {
+          member.h2 = undefined;
-          const optional = arg.name === 'options' || arg.text.includes('Optional');
-          if (tokens.length) {
-            if (optional && !hasOptional)
-              tokens.push(`[, ${arg.name}`);
-            else
-              tokens.push(`, ${arg.name}`);
-          } else {
-            if (optional && !hasOptional)
-              tokens.push(`[${arg.name}`);
-            else
-              tokens.push(`${arg.name}`);
-          }
-          hasOptional = hasOptional || optional;
-        }
-        if (hasOptional)
-          tokens.push(']');
-        const signature = tokens.join('');
-        signatures.set(lastMethod.h4, signature);
-        lastMethod.h4 = `${lastMethod.h4}(${signature})`;
-        lastMethod = null;
-        args = null;
-      };
-      for (const node of nodes) {
-        if (node.h1 || node.h2 || node.h3 || node.h4)
-          flushMethodSignature();
 
-        if (node.h4) {
+          let match = member.h4.match(/(event|method|namespace): (JS|CDP|[A-Z])([^.]+)\.(.*)/);
-          lastMethod = null;
-          args = null;
-          let match = node.h4.match(/(event|method|namespace) (JS|CDP|[A-Z])([^.]+)\.(.*)/);
           if (!match)
             continue;
+
           if (match[1] === 'event') {
-            node.h4 = `${match[2].toLowerCase() + match[3]}.on('${match[4]}')`;
+            member.h4 = `${match[2].toLowerCase() + match[3]}.on('${match[4]}')`;
             continue;
           }
 
           if (match[1] === 'method') {
-            node.h4 = `${match[2].toLowerCase() + match[3]}.${match[4]}`;
+            const args = [];
-            lastMethod = node;
+            const argChildren = [];
-            args = [];
+            const nonArgChildren = [];
-            continue;
+            const optionsContainer = [];
+            for (const item of member.children) {
+              if (!item.h3) {
+                nonArgChildren.push(item);
+                continue;
+              }
+              if (item.h3.startsWith('param:')) {
+                if (item.h3.includes('=')) {
+                  const [name, key] = item.h3.split(' = ');
+                  item.h3 = name;
+                  const template = paramsMap.get(key);
+                  if (!template)
+                    throw new Error('Bad template: ' + kkey);
+                  args.push(parseArgument(template.li));
+                  argChildren.push(template);
+                } else {
+                  const param = item.children[0];
+                  if (item.children[1])
+                    param.li += ' ' + item.children[1].text;
+                  args.push(parseArgument(param.li));
+                  argChildren.push(param);
+                }
+              }
+              if (item.h3.startsWith('option:')) {
+                let optionsNode = optionsContainer[0];
+                if (!optionsNode) {
+                  optionsNode = {
+                    li: '`options` <[Object]>',
+                    liType: 'default',
+                    children: [],
+                  };
+                  optionsContainer.push(optionsNode);
+                  args.push(parseArgument(optionsNode.li));
+                }
+                if (item.h3.includes('=')) {
+                  const [name, key] = item.h3.split(' = ');
+                  const template = paramsMap.get(key);
+                  if (!template)
+                    throw new Error('Bad template: ' + key);
+                  if (item.h3.includes('-inline-')) {
+                    optionsNode.children.push(...template.children);
+                  } else {
+                    item.h3 = name;
+                    optionsNode.children.push(template);
+                  }
+                } else {
+                  const param = item.children[0];
+                  if (item.children[1])
+                    param.li += ' ' + item.children[1].text;
+                  optionsNode.children.push(param);
+                }
+              }
+            }
+            member.children = [...argChildren, ...optionsContainer, ...nonArgChildren];
+
+            const tokens = [];
+            let hasOptional = false;
+            for (const arg of args) {
+              const optional = arg.name === 'options' || arg.text.includes('Optional');
+              if (tokens.length) {
+                if (optional && !hasOptional)
+                  tokens.push(`[, ${arg.name}`);
+                else
+                  tokens.push(`, ${arg.name}`);
+              } else {
+                if (optional && !hasOptional)
+                  tokens.push(`[${arg.name}`);
+                else
+                  tokens.push(`${arg.name}`);
+              }
+              hasOptional = hasOptional || optional;
+            }
+            if (hasOptional)
+              tokens.push(']');
+            const signature = tokens.join('');
+            const methodName = `${match[2].toLowerCase() + match[3]}.${match[4]}`;
+            signatures.set(methodName, signature);
+            member.h4 = `${methodName}(${signature})`;
           }
 
           if (match[1] === 'namespace') {
-            node.h4 = `${match[2].toLowerCase() + match[3]}.${match[4]}`;
+            member.h4 = `${match[2].toLowerCase() + match[3]}.${match[4]}`;
             continue;
           }
-
-          continue;
         }
-        if (args && node.li && node.liType === 'default' && !node.li.startsWith('returns'))
-          args.push(parseArgument(node.li));
       }
       api.setText([comment, header, renderMd(nodes), footer].join('\n'));
 
@@ -129,12 +192,6 @@ async function run() {
 
   // Documentation checks.
   {
-    const readme = await Source.readFile(path.join(PROJECT_DIR, 'README.md'));
-    const binReadme = await Source.readFile(path.join(PROJECT_DIR, 'bin', 'README.md'));
-    const contributing = await Source.readFile(path.join(PROJECT_DIR, 'CONTRIBUTING.md'));
-    const docs = await Source.readdir(path.join(PROJECT_DIR, 'docs'), '.md');
-    const mdSources = [readme, binReadme, api, contributing, ...docs];
-
     const browserVersions = await getBrowserVersions();
     messages.push(...(await preprocessor.runCommands(mdSources, {
       libversion: VERSION,
@@ -143,7 +200,6 @@ async function run() {
     })));
 
     messages.push(...preprocessor.autocorrectInvalidLinks(PROJECT_DIR, mdSources, getRepositoryFiles()));
-
     for (const source of mdSources.filter(source => source.hasUpdatedText()))
       messages.push(Message.warning(`WARN: updated ${source.projectPath()}`));
 

utils/parse_md.js

@@ -51,9 +51,12 @@ function normalizeLines(content) {
 
 function buildTree(lines) {
   const root = {
-    ul: []
+    h0: '<root>',
+    children: []
   };
   const stack = [root];
+  let liStack = null;
+
   for (let i = 0; i < lines.length; ++i) {
     let line = lines[i];
 
@@ -62,7 +65,7 @@ function buildTree(lines) {
         code: [],
         codeLang: line.substring(3)
       };
-      root.ul.push(node);
+      stack[0].children.push(node);
       line = lines[++i];
       while (!line.startsWith('```')) {
         node.code.push(line);
@@ -75,7 +78,7 @@ function buildTree(lines) {
       const node = {
         gen: [line]
       };
-      root.ul.push(node);
+      stack[0].children.push(node);
       line = lines[++i];
       while (!line.startsWith('<!-- GEN')) {
         node.gen.push(line);
@@ -87,9 +90,20 @@ function buildTree(lines) {
 
     const header = line.match(/^(#+)/);
     if (header) {
-      const node = {};
+      const node = { children: [] };
-      node['h' + header[1].length] = line.substring(header[1].length + 1);
+      const h = header[1].length;
-      root.ul.push(node);
+      node['h' + h] = line.substring(h + 1);
+
+      while (true) {
+        const lastH = +Object.keys(stack[0]).find(k => k.startsWith('h')).substring(1);
+        if (h <= lastH)
+          stack.shift();
+        else
+          break;
+      }
+      stack[0].children.push(node);
+      stack.unshift(node);
+      liStack = [node];
       continue;
     }
 
@@ -107,12 +121,12 @@ function buildTree(lines) {
     } else {
       node.text = line;
     }
-    if (!stack[depth].ul)
+    if (!liStack[depth].children)
-      stack[depth].ul = [];
+      liStack[depth].children = [];
-    stack[depth].ul.push(node);
+    liStack[depth].children.push(node);
-    stack[depth + 1] = node;
+    liStack[depth + 1] = node;
   }
-  return root.ul;
+  return root.children;
 }
 
 function parseMd(content) {
@@ -141,28 +155,26 @@ function innerRenderMdNode(node, lastNode, result) {
       result.push('');
   };
 
-  if (node.h1) {
+  for (let i = 1; i < 10; ++i) {
-    newLine();
+    if (node[`h${i}`]) {
-    result.push(`# ${node.h1}`);
+      newLine();
-  }
+      result.push(`${'#'.repeat(i)} ${node[`h${i}`]}`);
-  if (node.h2) {
+      let lastNode = node;
-    newLine();
+      for (const child of node.children) {
-    result.push(`## ${node.h2}`);
+        innerRenderMdNode(child, lastNode, result);
-  }
+        lastNode = child;
-  if (node.h3) {
+      }
-    newLine();
+      break;
-    result.push(`### ${node.h3}`);
+    }
-  }
-  if (node.h4) {
-    newLine();
-    result.push(`#### ${node.h4}`);
   }
+
   if (node.text) {
     const bothComments = node.text.startsWith('>') && lastNode && lastNode.text && lastNode.text.startsWith('>');
     if (!bothComments && lastNode && (lastNode.text || lastNode.li || lastNode.h1 || lastNode.h2 || lastNode.h3 || lastNode.h4))
       newLine();
       printText(node, result);
   }
+
   if (node.code) {
     newLine();
     result.push('```' + node.codeLang);
@@ -171,12 +183,14 @@ function innerRenderMdNode(node, lastNode, result) {
     result.push('```');
     newLine();
   }
+
   if (node.gen) {
     newLine();
     for (const line of node.gen)
       result.push(line);
     newLine();
   }
+
   if (node.li) {
     const visit = (node, indent) => {
       let char;
@@ -186,7 +200,7 @@ function innerRenderMdNode(node, lastNode, result) {
         case 'ordinal': char = '1.'; break;
       }
       result.push(`${indent}${char} ${node.li}`);
-      for (const child of node.ul || [])
+      for (const child of node.children || [])
         visit(child, indent + '  ');
     };
     visit(node, '');
@@ -209,73 +223,6 @@ function printText(node, result) {
     result.push(line);
 }
 
-function renderMdTemplate(body, params) {
-  const map = new Map();
-  let nodes;
-  for (const node of parseMd(params)) {
-    if (node.h2) {
-      const name = node.h2;
-      nodes = [];
-      map.set(name, nodes);
-      continue;
-    }
-    nodes.push(node);
-  }
-
-  const result = [];
-  for (const line of body.split('\n')) {
-    const match = line.match(/^(\s*)- %%-(.*)-%%/);
-    if (!match) {
-      result.push(line);
-      continue;
-    }
-    const indent = match[1];
-    const key = match[2];
-    const nodes = map.get(key);
-    if (!nodes)
-      throw new Error(`Missing param "${key}"`);
-
-    let snippet;
-    if (line.endsWith('-as-is')) {
-      snippet = nodes.map(node => renderMdNode(node)).join('\n');
-    } else {
-      const { name, type } = parseArgument(nodes[0].li);
-      nodes[0].li = `\`${name}\` ${type}`;
-      if (nodes[1])
-        nodes[0].li += ` ${nodes[1].text}`;
-      snippet = renderMdNode(nodes[0]);
-    }
-    for (const l of snippet.split('\n'))
-      result.push(indent + l);
-  }
-  return result.join('\n');
-}
-
-function extractParamDescriptions(params) {
-  let name;
-
-  for (const node of parseMd(params)) {
-    if (node.h2) {
-      name = node.h2;
-      continue;
-    }
-    extractParamDescription(name, node);
-  }
-}
-
-function extractParamDescription(group, node) {
-  const { name, type, text } = parseArgument(node.li);
-  node.li = `\`${name}\` ${type}`;
-  if (group === 'shared-context-params')
-    group = `context-option-${name.toLowerCase()}`;
-  console.log(`## ${group}`);
-  console.log();
-  console.log(renderMdNode(node));
-  console.log();
-  console.log(text);
-  console.log();
-}
-
 function parseArgument(line) {
   const match = line.match(/`([^`]+)` (.*)/);
   if (!match)
@@ -297,4 +244,4 @@ function parseArgument(line) {
   throw new Error('Should not be reached');
 }
 
-module.exports = { parseMd, renderMd, renderMdTemplate, extractParamDescriptions, parseArgument };
+module.exports = { parseMd, renderMd, parseArgument };