diff --git a/packages/playwright-core/src/web/htmlReport/htmlReport.tsx b/packages/playwright-core/src/web/htmlReport/htmlReport.tsx index 4a2422ee03..4dd6d02ef8 100644 --- a/packages/playwright-core/src/web/htmlReport/htmlReport.tsx +++ b/packages/playwright-core/src/web/htmlReport/htmlReport.tsx @@ -214,7 +214,7 @@ const TestResultView: React.FC<{ )} - {!!otherAttachments &&
Attachments
} + {!!otherAttachments.length &&
Attachments
} {otherAttachments.map((a, i) => )} ; }; diff --git a/packages/playwright-test/src/reporters/html.ts b/packages/playwright-test/src/reporters/html.ts index 55d6d8f333..7a4020ab98 100644 --- a/packages/playwright-test/src/reporters/html.ts +++ b/packages/playwright-test/src/reporters/html.ts @@ -249,9 +249,12 @@ class HtmlBuilder { status: result.status, attachments: result.attachments.map(a => { if (a.path) { - const fileName = 'data/' + test.testId + path.extname(a.path); + let fileName = a.path; try { - fs.copyFileSync(a.path, path.join(this._reportFolder, fileName)); + const buffer = fs.readFileSync(a.path); + const sha1 = calculateSha1(buffer) + path.extname(a.path); + fileName = 'data/' + sha1; + fs.writeFileSync(path.join(this._reportFolder, 'data', sha1), buffer); } catch (e) { } return { diff --git a/tests/config/baseTest.ts b/tests/config/baseTest.ts index 61219a2e47..04a1791b81 100644 --- a/tests/config/baseTest.ts +++ b/tests/config/baseTest.ts @@ -14,18 +14,15 @@ * limitations under the License. */ -import { TestServer } from '../../utils/testserver'; import { Fixtures, _baseTest } from '@playwright/test'; import * as path from 'path'; import * as fs from 'fs'; -import socks from 'socksv5'; import { installCoverageHooks } from './coverage'; import * as childProcess from 'child_process'; import { start } from 'playwright-core/lib/outofprocess'; import { PlaywrightClient } from 'playwright-core/lib/remote/playwrightClient'; import type { LaunchOptions } from 'playwright-core'; -import { TestProxy } from './proxy'; -import { commonFixtures, CommonFixtures } from './commonFixtures'; +import { commonFixtures, CommonFixtures, serverFixtures, ServerFixtures, ServerOptions } from './commonFixtures'; export type BrowserName = 'chromium' | 'firefox' | 'webkit'; type Mode = 'default' | 'driver' | 'service'; @@ -127,96 +124,6 @@ const baseFixtures: Fixtures<{}, BaseOptions & BaseFixtures> = { isLinux: [ process.platform === 'linux', { scope: 'worker' } ], }; -type ServerOptions = { - loopback?: string; -}; -export type ServerFixtures = { - server: TestServer; - httpsServer: TestServer; - socksPort: number; - proxyServer: TestProxy; - asset: (p: string) => string; -}; - -export type ServersInternal = ServerFixtures & { socksServer: socks.SocksServer }; -export const serverFixtures: Fixtures = { - loopback: [ undefined, { scope: 'worker' } ], - __servers: [ async ({ loopback }, run, workerInfo) => { - const assetsPath = path.join(__dirname, '..', 'assets'); - const cachedPath = path.join(__dirname, '..', 'assets', 'cached'); - - const port = 8907 + workerInfo.workerIndex * 4; - const server = await TestServer.create(assetsPath, port, loopback); - server.enableHTTPCache(cachedPath); - - const httpsPort = port + 1; - const httpsServer = await TestServer.createHTTPS(assetsPath, httpsPort, loopback); - httpsServer.enableHTTPCache(cachedPath); - - const socksServer = socks.createServer((info, accept, deny) => { - const socket = accept(true); - if (socket) { - // Catch and ignore ECONNRESET errors. - socket.on('error', () => {}); - const body = 'Served by the SOCKS proxy'; - socket.end([ - 'HTTP/1.1 200 OK', - 'Connection: close', - 'Content-Type: text/html', - 'Content-Length: ' + Buffer.byteLength(body), - '', - body - ].join('\r\n')); - } - }); - const socksPort = port + 2; - socksServer.listen(socksPort, 'localhost'); - socksServer.useAuth(socks.auth.None()); - - const proxyPort = port + 3; - const proxyServer = await TestProxy.create(proxyPort); - - await run({ - asset: (p: string) => path.join(__dirname, '..', 'assets', ...p.split('/')), - server, - httpsServer, - socksPort, - proxyServer, - socksServer, - }); - - await Promise.all([ - server.stop(), - httpsServer.stop(), - socksServer.close(), - proxyServer.stop(), - ]); - }, { scope: 'worker' } ], - - server: async ({ __servers }, run) => { - __servers.server.reset(); - await run(__servers.server); - }, - - httpsServer: async ({ __servers }, run) => { - __servers.httpsServer.reset(); - await run(__servers.httpsServer); - }, - - socksPort: async ({ __servers }, run) => { - await run(__servers.socksPort); - }, - - proxyServer: async ({ __servers }, run) => { - __servers.proxyServer.reset(); - await run(__servers.proxyServer); - }, - - asset: async ({ __servers }, run) => { - await run(__servers.asset); - }, -}; - type CoverageOptions = { coverageName?: string; }; diff --git a/tests/config/commonFixtures.ts b/tests/config/commonFixtures.ts index 519d08511d..0876409e86 100644 --- a/tests/config/commonFixtures.ts +++ b/tests/config/commonFixtures.ts @@ -15,8 +15,12 @@ */ import type { Fixtures } from '@playwright/test'; -import { spawn, ChildProcess, execSync } from 'child_process'; +import { ChildProcess, execSync, spawn } from 'child_process'; import net from 'net'; +import path from 'path'; +import socks from 'socksv5'; +import { TestServer } from '../../utils/testserver'; +import { TestProxy } from './proxy'; type TestChildParams = { command: string[], @@ -146,3 +150,93 @@ export const commonFixtures: Fixtures = { token.canceled = true; }, }; + +export type ServerOptions = { + loopback?: string; +}; +export type ServerFixtures = { + server: TestServer; + httpsServer: TestServer; + socksPort: number; + proxyServer: TestProxy; + asset: (p: string) => string; +}; + +export type ServersInternal = ServerFixtures & { socksServer: socks.SocksServer }; +export const serverFixtures: Fixtures = { + loopback: [ undefined, { scope: 'worker' } ], + __servers: [ async ({ loopback }, run, workerInfo) => { + const assetsPath = path.join(__dirname, '..', 'assets'); + const cachedPath = path.join(__dirname, '..', 'assets', 'cached'); + + const port = 8907 + workerInfo.workerIndex * 4; + const server = await TestServer.create(assetsPath, port, loopback); + server.enableHTTPCache(cachedPath); + + const httpsPort = port + 1; + const httpsServer = await TestServer.createHTTPS(assetsPath, httpsPort, loopback); + httpsServer.enableHTTPCache(cachedPath); + + const socksServer = socks.createServer((info, accept, deny) => { + const socket = accept(true); + if (socket) { + // Catch and ignore ECONNRESET errors. + socket.on('error', () => {}); + const body = 'Served by the SOCKS proxy'; + socket.end([ + 'HTTP/1.1 200 OK', + 'Connection: close', + 'Content-Type: text/html', + 'Content-Length: ' + Buffer.byteLength(body), + '', + body + ].join('\r\n')); + } + }); + const socksPort = port + 2; + socksServer.listen(socksPort, 'localhost'); + socksServer.useAuth(socks.auth.None()); + + const proxyPort = port + 3; + const proxyServer = await TestProxy.create(proxyPort); + + await run({ + asset: (p: string) => path.join(__dirname, '..', 'assets', ...p.split('/')), + server, + httpsServer, + socksPort, + proxyServer, + socksServer, + }); + + await Promise.all([ + server.stop(), + httpsServer.stop(), + socksServer.close(), + proxyServer.stop(), + ]); + }, { scope: 'worker' } ], + + server: async ({ __servers }, run) => { + __servers.server.reset(); + await run(__servers.server); + }, + + httpsServer: async ({ __servers }, run) => { + __servers.httpsServer.reset(); + await run(__servers.httpsServer); + }, + + socksPort: async ({ __servers }, run) => { + await run(__servers.socksPort); + }, + + proxyServer: async ({ __servers }, run) => { + __servers.proxyServer.reset(); + await run(__servers.proxyServer); + }, + + asset: async ({ __servers }, run) => { + await run(__servers.asset); + }, +}; diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.ts index 9b9f3ab293..f08ec967e9 100644 --- a/tests/playwright-test/playwright-test-fixtures.ts +++ b/tests/playwright-test/playwright-test-fixtures.ts @@ -14,15 +14,14 @@ * limitations under the License. */ -import { TestInfo, test as base } from './stable-test-runner'; -import { CommonFixtures, commonFixtures } from '../config/commonFixtures'; -import * as fs from 'fs'; -import * as path from 'path'; -import * as os from 'os'; import type { JSONReport, JSONReportSuite } from '@playwright/test/src/reporters/json'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; import rimraf from 'rimraf'; import { promisify } from 'util'; -import { serverFixtures, ServerFixtures } from '../config/baseTest'; +import { CommonFixtures, commonFixtures, serverFixtures, ServerFixtures } from '../config/commonFixtures'; +import { test as base, TestInfo } from './stable-test-runner'; const removeFolderAsync = promisify(rimraf); diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index 77d6af2d64..687d9317a6 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -16,9 +16,29 @@ import fs from 'fs'; import path from 'path'; -import { test, expect } from './playwright-test-fixtures'; +import { test as baseTest, expect } from './playwright-test-fixtures'; +import { HttpServer } from 'playwright-core/src/utils/httpServer'; -const kHTMLReporterPath = path.join(__dirname, '..', '..', 'packages', 'playwright-test', 'lib', 'reporters', 'html.js'); +const test = baseTest.extend<{ showReport: () => Promise }>({ + showReport: async ({ page }, use, testInfo) => { + const server = new HttpServer(); + await use(async () => { + const reportFolder = testInfo.outputPath('playwright-report'); + server.routePrefix('/', (request, response) => { + let relativePath = new URL('http://localhost' + request.url).pathname; + if (relativePath === '/') + relativePath = '/index.html'; + const absolutePath = path.join(reportFolder, ...relativePath.split('/')); + return server.serveFile(response, absolutePath); + }); + const location = await server.start(); + await page.goto(location); + }); + await server.stop(); + } +}); + +test.use({ channel: 'chrome' }); test('should generate report', async ({ runInlineTest }, testInfo) => { await runInlineTest({ @@ -38,7 +58,7 @@ test('should generate report', async ({ runInlineTest }, testInfo) => { expect(testInfo.retry).toBe(1); }); `, - }, { reporter: 'dot,' + kHTMLReporterPath, retries: 1 }); + }, { reporter: 'dot,html', retries: 1 }); const report = testInfo.outputPath('playwright-report', 'data', 'projects.json'); const reportObject = JSON.parse(fs.readFileSync(report, 'utf-8')); delete reportObject[0].suites[0].duration; @@ -138,7 +158,73 @@ test('should not throw when attachment is missing', async ({ runInlineTest }) => testInfo.attachments.push({ name: 'screenshot', path: screenshot, contentType: 'image/png' }); }); `, - }, { reporter: 'dot,' + kHTMLReporterPath }); + }, { reporter: 'dot,html' }); expect(result.exitCode).toBe(0); expect(result.passed).toBe(1); }); + +test('should include image diff', async ({ runInlineTest, page, showReport }) => { + const expected = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAMgAAADICAYAAACtWK6eAAAAAXNSR0IArs4c6QAAAhVJREFUeJzt07ERwCAQwLCQ/Xd+FuDcQiFN4MZrZuYDjv7bAfAyg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAiEDVPZBYx6ffy+AAAAAElFTkSuQmCC', 'base64'); + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { use: { viewport: { width: 200, height: 200 }} }; + `, + 'a.test.js-snapshots/expected-linux.png': expected, + 'a.test.js-snapshots/expected-darwin.png': expected, + 'a.test.js-snapshots/expected-win32.png': expected, + 'a.test.js': ` + const { test } = pwt; + test('fails', async ({ page }, testInfo) => { + await page.setContent('Hello World'); + const screenshot = await page.screenshot(); + await expect(screenshot).toMatchSnapshot('expected.png'); + }); + `, + }, { reporter: 'dot,html' }); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + + await showReport(); + await page.click('text=a.test.js'); + await page.click('text=fails'); + const imageDiff = page.locator('.test-image-mismatch'); + const image = imageDiff.locator('img'); + await expect(image).toHaveAttribute('src', /.*png/); + const actualSrc = await image.getAttribute('src'); + await imageDiff.locator('text=Expected').click(); + const expectedSrc = await image.getAttribute('src'); + await imageDiff.locator('text=Diff').click(); + const diffSrc = await image.getAttribute('src'); + const set = new Set([expectedSrc, actualSrc, diffSrc]); + expect(set.size).toBe(3); +}); + +test('should include screenshot on failure', async ({ runInlineTest, page, showReport }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + use: { + viewport: { width: 200, height: 200 }, + screenshot: 'only-on-failure', + } + }; + `, + 'a.test.js': ` + const { test } = pwt; + test('fails', async ({ page }) => { + await page.setContent('Failed state'); + await expect(true).toBeFalsy(); + }); + `, + }, { reporter: 'dot,html' }); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + + await showReport(); + await page.click('text=a.test.js'); + await page.click('text=fails'); + await expect(page.locator('text=Screenshots')).toBeVisible(); + await expect(page.locator('img')).toBeVisible(); + const src = await page.locator('img').getAttribute('src'); + expect(src).toBeTruthy(); +});