From 2f8d448dbbcc9130703035203f469fae26b808d6 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Mon, 10 Feb 2025 15:02:19 +0100 Subject: [PATCH] feat(html): "copy prompt" button (#34670) --- packages/html-reporter/src/metadataView.tsx | 31 ++++++++--- packages/html-reporter/src/reportView.tsx | 5 +- packages/html-reporter/src/testErrorView.css | 33 +++++++++++- packages/html-reporter/src/testErrorView.tsx | 52 ++++++++++++++++-- packages/html-reporter/src/testFilesView.tsx | 6 +-- packages/html-reporter/src/testResultView.tsx | 6 +-- packages/playwright/src/reporters/html.ts | 11 ++++ packages/web/src/components/prompts.ts | 48 +++++++++++++++++ tests/playwright-test/reporter-html.spec.ts | 53 +++++++++++++++++++ 9 files changed, 223 insertions(+), 22 deletions(-) create mode 100644 packages/web/src/components/prompts.ts diff --git a/packages/html-reporter/src/metadataView.tsx b/packages/html-reporter/src/metadataView.tsx index 8cc66571ab..ed050f4480 100644 --- a/packages/html-reporter/src/metadataView.tsx +++ b/packages/html-reporter/src/metadataView.tsx @@ -26,9 +26,25 @@ import { linkifyText } from '@web/renderUtils'; type MetadataEntries = [string, unknown][]; -export function filterMetadata(metadata: Metadata): MetadataEntries { - // TODO: do not plumb actualWorkers through metadata. - return Object.entries(metadata).filter(([key]) => key !== 'actualWorkers'); +export const MetadataContext = React.createContext([]); + +export function MetadataProvider({ metadata, children }: React.PropsWithChildren<{ metadata: Metadata }>) { + const entries = React.useMemo(() => { + // TODO: do not plumb actualWorkers through metadata. + + return Object.entries(metadata).filter(([key]) => key !== 'actualWorkers'); + }, [metadata]); + + return {children}; +} + +export function useMetadata() { + return React.useContext(MetadataContext); +} + +export function useGitCommitInfo() { + const metadataEntries = useMetadata(); + return metadataEntries.find(([key]) => key === 'git.commit.info')?.[1] as GitCommitInfo | undefined; } class ErrorBoundary extends React.Component, { error: Error | null, errorInfo: React.ErrorInfo | null }> { @@ -57,12 +73,13 @@ class ErrorBoundary extends React.Component, { error } } -export const MetadataView: React.FC<{ metadataEntries: MetadataEntries }> = ({ metadataEntries }) => { - return ; +export const MetadataView = () => { + return ; }; -const InnerMetadataView: React.FC<{ metadataEntries: MetadataEntries }> = ({ metadataEntries }) => { - const gitCommitInfo = metadataEntries.find(([key]) => key === 'git.commit.info')?.[1] as GitCommitInfo | undefined; +const InnerMetadataView = () => { + const metadataEntries = useMetadata(); + const gitCommitInfo = useGitCommitInfo(); const entries = metadataEntries.filter(([key]) => key !== 'git.commit.info'); if (!gitCommitInfo && !entries.length) return null; diff --git a/packages/html-reporter/src/reportView.tsx b/packages/html-reporter/src/reportView.tsx index e48064201c..baf85f32a8 100644 --- a/packages/html-reporter/src/reportView.tsx +++ b/packages/html-reporter/src/reportView.tsx @@ -26,6 +26,7 @@ import './reportView.css'; import { TestCaseView } from './testCaseView'; import { TestFilesHeader, TestFilesView } from './testFilesView'; import './theme.css'; +import { MetadataProvider } from './metadataView'; declare global { interface Window { @@ -72,7 +73,7 @@ export const ReportView: React.FC<{ return result; }, [report, filter]); - return
+ return
{report?.json() && } @@ -88,7 +89,7 @@ export const ReportView: React.FC<{ {!!report && }
-
; +
; }; const TestCaseViewLoader: React.FC<{ diff --git a/packages/html-reporter/src/testErrorView.css b/packages/html-reporter/src/testErrorView.css index e29ea2a18b..7cc9e88a79 100644 --- a/packages/html-reporter/src/testErrorView.css +++ b/packages/html-reporter/src/testErrorView.css @@ -16,18 +16,47 @@ @import '@web/third_party/vscode/colors.css'; -.test-error-view { +.test-error-container { white-space: pre; overflow: auto; flex: none; padding: 0; background-color: var(--color-canvas-subtle); border-radius: 6px; - padding: 16px; line-height: initial; margin-bottom: 6px; } +.test-error-view { + padding: 16px; +} + .test-error-text { font-family: monospace; } + +.prompt-button { + flex: none; + height: 24px; + width: 80px; + border: 1px solid var(--color-btn-border); + outline: none; + color: var(--color-btn-text); + background: var(--color-btn-bg); + padding: 4px; + cursor: pointer; + display: inline-flex; + align-items: center; + justify-content: center; + border-radius: 4px; +} + +.prompt-button svg { + color: var(--color-fg-subtle); +} + +.prompt-button:not(:disabled):hover { + border-color: var(--color-btn-hover-border); + background-color: var(--color-btn-hover-bg); +} + diff --git a/packages/html-reporter/src/testErrorView.tsx b/packages/html-reporter/src/testErrorView.tsx index ea402106a8..11f8e4b474 100644 --- a/packages/html-reporter/src/testErrorView.tsx +++ b/packages/html-reporter/src/testErrorView.tsx @@ -17,15 +17,57 @@ import { ansi2html } from '@web/ansi2html'; import * as React from 'react'; import './testErrorView.css'; +import * as icons from './icons'; import type { ImageDiff } from '@web/shared/imageDiffView'; import { ImageDiffView } from '@web/shared/imageDiffView'; +import type { TestResult } from './types'; +import { fixTestPrompt } from '@web/components/prompts'; +import { useGitCommitInfo } from './metadataView'; -export const TestErrorView: React.FC<{ +export const TestErrorView: React.FC<{ error: string; testId?: string; result?: TestResult }> = ({ error, testId, result }) => { + return ( + +
+ +
+ + ); +}; + +export const CodeSnippet = ({ code, children, testId }: React.PropsWithChildren<{ code: string; testId?: string; }>) => { + const html = React.useMemo(() => ansiErrorToHtml(code), [code]); + return ( +
+ {children} +
+
+ ); +}; + +const PromptButton: React.FC<{ error: string; - testId?: string; -}> = ({ error, testId }) => { - const html = React.useMemo(() => ansiErrorToHtml(error), [error]); - return
; + result?: TestResult; +}> = ({ error, result }) => { + const gitCommitInfo = useGitCommitInfo(); + const prompt = React.useMemo(() => fixTestPrompt( + error, + gitCommitInfo?.['pull.diff'] ?? gitCommitInfo?.['revision.diff'], + result?.attachments.find(a => a.name === 'pageSnapshot')?.body + ), [gitCommitInfo, result, error]); + + const [copied, setCopied] = React.useState(false); + + return ; }; export const TestScreenshotErrorView: React.FC<{ diff --git a/packages/html-reporter/src/testFilesView.tsx b/packages/html-reporter/src/testFilesView.tsx index bcb0696946..dcac5e8cea 100644 --- a/packages/html-reporter/src/testFilesView.tsx +++ b/packages/html-reporter/src/testFilesView.tsx @@ -22,7 +22,7 @@ import { msToString } from './utils'; import { AutoChip } from './chip'; import { TestErrorView } from './testErrorView'; import * as icons from './icons'; -import { filterMetadata, MetadataView } from './metadataView'; +import { MetadataView, useMetadata } from './metadataView'; export const TestFilesView: React.FC<{ tests: TestFileSummary[], @@ -67,9 +67,9 @@ export const TestFilesHeader: React.FC<{ metadataVisible: boolean, toggleMetadataVisible: () => void, }> = ({ report, filteredStats, metadataVisible, toggleMetadataVisible }) => { + const metadataEntries = useMetadata(); if (!report) return; - const metadataEntries = filterMetadata(report.metadata || {}); return <>
{metadataEntries.length > 0 &&
@@ -81,7 +81,7 @@ export const TestFilesHeader: React.FC<{
{report ? new Date(report.startTime).toLocaleString() : ''}
Total time: {msToString(report.duration ?? 0)}
- {metadataVisible && } + {metadataVisible && } {!!report.errors.length && {report.errors.map((error, index) => )} } diff --git a/packages/html-reporter/src/testResultView.tsx b/packages/html-reporter/src/testResultView.tsx index 681f4b507a..3243b2bcb9 100644 --- a/packages/html-reporter/src/testResultView.tsx +++ b/packages/html-reporter/src/testResultView.tsx @@ -24,7 +24,7 @@ import { Anchor, AttachmentLink, generateTraceUrl, testResultHref } from './link import { statusIcon } from './statusIcon'; import type { ImageDiff } from '@web/shared/imageDiffView'; import { ImageDiffView } from '@web/shared/imageDiffView'; -import { TestErrorView, TestScreenshotErrorView } from './testErrorView'; +import { CodeSnippet, TestErrorView, TestScreenshotErrorView } from './testErrorView'; import * as icons from './icons'; import './testResultView.css'; @@ -90,7 +90,7 @@ export const TestResultView: React.FC<{ {errors.map((error, index) => { if (error.type === 'screenshot') return ; - return ; + return ; })} } {!!result.steps.length && @@ -182,7 +182,7 @@ const StepTreeItem: React.FC<{ {step.count > 1 && <> ✕ {step.count}} {step.location && — {step.location.file}:{step.location.line}} } loadChildren={step.steps.length || step.snippet ? () => { - const snippet = step.snippet ? [] : []; + const snippet = step.snippet ? [] : []; const steps = step.steps.map((s, i) => ); return snippet.concat(steps); } : undefined} depth={depth}/>; diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts index 557ad580b7..2600b51797 100644 --- a/packages/playwright/src/reporters/html.ts +++ b/packages/playwright/src/reporters/html.ts @@ -449,6 +449,17 @@ class HtmlBuilder { return a; } + if (a.name === 'pageSnapshot') { + try { + const body = fs.readFileSync(a.path!, { encoding: 'utf-8' }); + return { + name: 'pageSnapshot', + contentType: a.contentType, + body, + }; + } catch {} + } + if (a.path) { let fileName = a.path; try { diff --git a/packages/web/src/components/prompts.ts b/packages/web/src/components/prompts.ts new file mode 100644 index 0000000000..88a086815e --- /dev/null +++ b/packages/web/src/components/prompts.ts @@ -0,0 +1,48 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const ansiRegex = new RegExp('([\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~])))', 'g'); +function stripAnsiEscapes(str: string): string { + return str.replace(ansiRegex, ''); +} + +export function fixTestPrompt(error: string, diff?: string, pageSnapshot?: string) { + const promptParts = [ + 'This test failed, suggest how to fix it. Please be correct, concise and keep Playwright best practices in mind.', + 'Here is the error:', + '\n', + stripAnsiEscapes(error), + '\n', + ]; + + if (pageSnapshot) { + promptParts.push( + 'This is how the page looked at the end of the test:', + pageSnapshot, + '\n' + ); + } + + if (diff) { + promptParts.push( + 'And this is the code diff:', + diff, + '\n' + ); + } + + return promptParts.join('\n'); +} diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index d5cc7ed50f..acb54f3b31 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -2694,6 +2694,59 @@ for (const useIntermediateMergeReport of [true, false] as const) { await page.getByText('my test').click(); await expect(page.locator('.tree-item', { hasText: 'stdout' })).toHaveCount(1); }); + + test('should show AI prompt', async ({ runInlineTest, writeFiles, showReport, page }) => { + const files = { + 'uncommitted.txt': `uncommitted file`, + 'playwright.config.ts': ` + export default { + populateGitInfo: true, + metadata: { foo: 'value1', bar: { prop: 'value2' }, baz: ['value3', 123] } + }; + `, + 'example.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('sample', async ({}) => { expect(2).toBe(3); }); + `, + }; + const baseDir = await writeFiles(files); + + const execGit = async (args: string[]) => { + const { code, stdout, stderr } = await spawnAsync('git', args, { stdio: 'pipe', cwd: baseDir }); + if (!!code) + throw new Error(`Non-zero exit of:\n$ git ${args.join(' ')}\nConsole:\nstdout:\n${stdout}\n\nstderr:\n${stderr}\n\n`); + return; + }; + + await execGit(['init']); + await execGit(['config', '--local', 'user.email', 'shakespeare@example.local']); + await execGit(['config', '--local', 'user.name', 'William']); + await execGit(['add', 'playwright.config.ts']); + await execGit(['commit', '-m', 'init']); + await execGit(['add', '*.ts']); + await execGit(['commit', '-m', 'chore(html): make this test look nice']); + + const result = await runInlineTest(files, { reporter: 'dot,html' }, { + PLAYWRIGHT_HTML_OPEN: 'never', + GITHUB_REPOSITORY: 'microsoft/playwright-example-for-test', + GITHUB_RUN_ID: 'example-run-id', + GITHUB_SERVER_URL: 'https://playwright.dev', + GITHUB_SHA: 'example-sha', + GITHUB_REF_NAME: '42/merge', + GITHUB_BASE_REF: 'HEAD~1', + }); + + expect(result.exitCode).toBe(1); + await showReport(); + + await page.context().grantPermissions(['clipboard-read', 'clipboard-write']); + + await page.getByRole('link', { name: 'sample' }).click(); + await page.getByRole('button', { name: 'Fix with AI' }).click(); + const prompt = await page.evaluate(() => navigator.clipboard.readText()); + expect(prompt, 'contains error').toContain('expect(received).toBe(expected)'); + expect(prompt, 'contains diff').toContain(`+ test('sample', async ({}) => { expect(2).toBe(3); });`); + }); }); }