Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

chore: start adding html runner e2e tests (#9518)

Browse source
This commit is contained in:
Pavel Feldman 2021-10-14 14:48:05 -08:00 committed by GitHub
parent 4c9d7d5ccc
commit 75cfe5d1f5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 197 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/playwright-core/src/web/htmlReport/htmlReport.tsx
View file

@ -214,7 +214,7 @@ const TestResultView: React.FC<{
<AttachmentLink attachment={a}></AttachmentLink> <AttachmentLink attachment={a}></AttachmentLink>
</div>)} </div>)}
{!!otherAttachments && <div key='attachments-title' className='test-overview-title'>Attachments</div>} {!!otherAttachments.length && <div key='attachments-title' className='test-overview-title'>Attachments</div>}
{otherAttachments.map((a, i) => <AttachmentLink key={`attachment-link-${i}`} attachment={a}></AttachmentLink>)} {otherAttachments.map((a, i) => <AttachmentLink key={`attachment-link-${i}`} attachment={a}></AttachmentLink>)}
</div>; </div>;
}; };

7
packages/playwright-test/src/reporters/html.ts
View file

@ -249,9 +249,12 @@ class HtmlBuilder {
status: result.status, status: result.status,
attachments: result.attachments.map(a => { attachments: result.attachments.map(a => {
if (a.path) { if (a.path) {
const fileName = 'data/' + test.testId + path.extname(a.path); let fileName = a.path;
try { try {
fs.copyFileSync(a.path, path.join(this._reportFolder, fileName)); const buffer = fs.readFileSync(a.path);
const sha1 = calculateSha1(buffer) + path.extname(a.path);
fileName = 'data/' + sha1;
fs.writeFileSync(path.join(this._reportFolder, 'data', sha1), buffer);
} catch (e) { } catch (e) {
} }
return { return {

95
tests/config/baseTest.ts
View file

@ -14,18 +14,15 @@
* limitations under the License. * limitations under the License.
*/ */
import { TestServer } from '../../utils/testserver';
import { Fixtures, _baseTest } from '@playwright/test'; import { Fixtures, _baseTest } from '@playwright/test';
import * as path from 'path'; import * as path from 'path';
import * as fs from 'fs'; import * as fs from 'fs';
import socks from 'socksv5';
import { installCoverageHooks } from './coverage'; import { installCoverageHooks } from './coverage';
import * as childProcess from 'child_process'; import * as childProcess from 'child_process';
import { start } from 'playwright-core/lib/outofprocess'; import { start } from 'playwright-core/lib/outofprocess';
import { PlaywrightClient } from 'playwright-core/lib/remote/playwrightClient'; import { PlaywrightClient } from 'playwright-core/lib/remote/playwrightClient';
import type { LaunchOptions } from 'playwright-core'; import type { LaunchOptions } from 'playwright-core';
import { TestProxy } from './proxy'; import { commonFixtures, CommonFixtures, serverFixtures, ServerFixtures, ServerOptions } from './commonFixtures';
import { commonFixtures, CommonFixtures } from './commonFixtures';
export type BrowserName = 'chromium' | 'firefox' | 'webkit'; export type BrowserName = 'chromium' | 'firefox' | 'webkit';
type Mode = 'default' | 'driver' | 'service'; type Mode = 'default' | 'driver' | 'service';
@ -127,96 +124,6 @@ const baseFixtures: Fixtures<{}, BaseOptions & BaseFixtures> = {
isLinux: [ process.platform === 'linux', { scope: 'worker' } ], isLinux: [ process.platform === 'linux', { scope: 'worker' } ],
}; };
type ServerOptions = {
loopback?: string;
};
export type ServerFixtures = {
server: TestServer;
httpsServer: TestServer;
socksPort: number;
proxyServer: TestProxy;
asset: (p: string) => string;
};
export type ServersInternal = ServerFixtures & { socksServer: socks.SocksServer };
export const serverFixtures: Fixtures<ServerFixtures, ServerOptions & { __servers: ServersInternal }> = {
loopback: [ undefined, { scope: 'worker' } ],
__servers: [ async ({ loopback }, run, workerInfo) => {
const assetsPath = path.join(__dirname, '..', 'assets');
const cachedPath = path.join(__dirname, '..', 'assets', 'cached');
const port = 8907 + workerInfo.workerIndex * 4;
const server = await TestServer.create(assetsPath, port, loopback);
server.enableHTTPCache(cachedPath);
const httpsPort = port + 1;
const httpsServer = await TestServer.createHTTPS(assetsPath, httpsPort, loopback);
httpsServer.enableHTTPCache(cachedPath);
const socksServer = socks.createServer((info, accept, deny) => {
const socket = accept(true);
if (socket) {
// Catch and ignore ECONNRESET errors.
socket.on('error', () => {});
const body = '<html><title>Served by the SOCKS proxy</title></html>';
socket.end([
'HTTP/1.1 200 OK',
'Connection: close',
'Content-Type: text/html',
'Content-Length: ' + Buffer.byteLength(body),
'',
body
].join('\r\n'));
}
});
const socksPort = port + 2;
socksServer.listen(socksPort, 'localhost');
socksServer.useAuth(socks.auth.None());
const proxyPort = port + 3;
const proxyServer = await TestProxy.create(proxyPort);
await run({
asset: (p: string) => path.join(__dirname, '..', 'assets', ...p.split('/')),
server,
httpsServer,
socksPort,
proxyServer,
socksServer,
});
await Promise.all([
server.stop(),
httpsServer.stop(),
socksServer.close(),
proxyServer.stop(),
]);
}, { scope: 'worker' } ],
server: async ({ __servers }, run) => {
__servers.server.reset();
await run(__servers.server);
},
httpsServer: async ({ __servers }, run) => {
__servers.httpsServer.reset();
await run(__servers.httpsServer);
},
socksPort: async ({ __servers }, run) => {
await run(__servers.socksPort);
},
proxyServer: async ({ __servers }, run) => {
__servers.proxyServer.reset();
await run(__servers.proxyServer);
},
asset: async ({ __servers }, run) => {
await run(__servers.asset);
},
};
type CoverageOptions = { type CoverageOptions = {
coverageName?: string; coverageName?: string;
}; };

96
tests/config/commonFixtures.ts
View file

@ -15,8 +15,12 @@
*/ */
import type { Fixtures } from '@playwright/test'; import type { Fixtures } from '@playwright/test';
import { spawn, ChildProcess, execSync } from 'child_process'; import { ChildProcess, execSync, spawn } from 'child_process';
import net from 'net'; import net from 'net';
import path from 'path';
import socks from 'socksv5';
import { TestServer } from '../../utils/testserver';
import { TestProxy } from './proxy';
type TestChildParams = { type TestChildParams = {
command: string[], command: string[],
@ -146,3 +150,93 @@ export const commonFixtures: Fixtures<CommonFixtures, {}> = {
token.canceled = true; token.canceled = true;
}, },
}; };
export type ServerOptions = {
loopback?: string;
};
export type ServerFixtures = {
server: TestServer;
httpsServer: TestServer;
socksPort: number;
proxyServer: TestProxy;
asset: (p: string) => string;
};
export type ServersInternal = ServerFixtures & { socksServer: socks.SocksServer };
export const serverFixtures: Fixtures<ServerFixtures, ServerOptions & { __servers: ServersInternal }> = {
loopback: [ undefined, { scope: 'worker' } ],
__servers: [ async ({ loopback }, run, workerInfo) => {
const assetsPath = path.join(__dirname, '..', 'assets');
const cachedPath = path.join(__dirname, '..', 'assets', 'cached');
const port = 8907 + workerInfo.workerIndex * 4;
const server = await TestServer.create(assetsPath, port, loopback);
server.enableHTTPCache(cachedPath);
const httpsPort = port + 1;
const httpsServer = await TestServer.createHTTPS(assetsPath, httpsPort, loopback);
httpsServer.enableHTTPCache(cachedPath);
const socksServer = socks.createServer((info, accept, deny) => {
const socket = accept(true);
if (socket) {
// Catch and ignore ECONNRESET errors.
socket.on('error', () => {});
const body = '<html><title>Served by the SOCKS proxy</title></html>';
socket.end([
'HTTP/1.1 200 OK',
'Connection: close',
'Content-Type: text/html',
'Content-Length: ' + Buffer.byteLength(body),
'',
body
].join('\r\n'));
}
});
const socksPort = port + 2;
socksServer.listen(socksPort, 'localhost');
socksServer.useAuth(socks.auth.None());
const proxyPort = port + 3;
const proxyServer = await TestProxy.create(proxyPort);
await run({
asset: (p: string) => path.join(__dirname, '..', 'assets', ...p.split('/')),
server,
httpsServer,
socksPort,
proxyServer,
socksServer,
});
await Promise.all([
server.stop(),
httpsServer.stop(),
socksServer.close(),
proxyServer.stop(),
]);
}, { scope: 'worker' } ],
server: async ({ __servers }, run) => {
__servers.server.reset();
await run(__servers.server);
},
httpsServer: async ({ __servers }, run) => {
__servers.httpsServer.reset();
await run(__servers.httpsServer);
},
socksPort: async ({ __servers }, run) => {
await run(__servers.socksPort);
},
proxyServer: async ({ __servers }, run) => {
__servers.proxyServer.reset();
await run(__servers.proxyServer);
},
asset: async ({ __servers }, run) => {
await run(__servers.asset);
},
};

11
tests/playwright-test/playwright-test-fixtures.ts
View file

@ -14,15 +14,14 @@
* limitations under the License. * limitations under the License.
*/ */
import { TestInfo, test as base } from './stable-test-runner';
import { CommonFixtures, commonFixtures } from '../config/commonFixtures';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import type { JSONReport, JSONReportSuite } from '@playwright/test/src/reporters/json'; import type { JSONReport, JSONReportSuite } from '@playwright/test/src/reporters/json';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import rimraf from 'rimraf'; import rimraf from 'rimraf';
import { promisify } from 'util'; import { promisify } from 'util';
import { serverFixtures, ServerFixtures } from '../config/baseTest'; import { CommonFixtures, commonFixtures, serverFixtures, ServerFixtures } from '../config/commonFixtures';
import { test as base, TestInfo } from './stable-test-runner';
const removeFolderAsync = promisify(rimraf); const removeFolderAsync = promisify(rimraf);

94
tests/playwright-test/reporter-html.spec.ts
View file

@ -16,9 +16,29 @@
import fs from 'fs'; import fs from 'fs';
import path from 'path'; import path from 'path';
import { test, expect } from './playwright-test-fixtures'; import { test as baseTest, expect } from './playwright-test-fixtures';
import { HttpServer } from 'playwright-core/src/utils/httpServer';
const kHTMLReporterPath = path.join(__dirname, '..', '..', 'packages', 'playwright-test', 'lib', 'reporters', 'html.js'); const test = baseTest.extend<{ showReport: () => Promise<void> }>({
showReport: async ({ page }, use, testInfo) => {
const server = new HttpServer();
await use(async () => {
const reportFolder = testInfo.outputPath('playwright-report');
server.routePrefix('/', (request, response) => {
let relativePath = new URL('http://localhost' + request.url).pathname;
if (relativePath === '/')
relativePath = '/index.html';
const absolutePath = path.join(reportFolder, ...relativePath.split('/'));
return server.serveFile(response, absolutePath);
});
const location = await server.start();
await page.goto(location);
});
await server.stop();
}
});
test.use({ channel: 'chrome' });
test('should generate report', async ({ runInlineTest }, testInfo) => { test('should generate report', async ({ runInlineTest }, testInfo) => {
await runInlineTest({ await runInlineTest({
@ -38,7 +58,7 @@ test('should generate report', async ({ runInlineTest }, testInfo) => {
expect(testInfo.retry).toBe(1); expect(testInfo.retry).toBe(1);
}); });
`, `,
}, { reporter: 'dot,' + kHTMLReporterPath, retries: 1 }); }, { reporter: 'dot,html', retries: 1 });
const report = testInfo.outputPath('playwright-report', 'data', 'projects.json'); const report = testInfo.outputPath('playwright-report', 'data', 'projects.json');
const reportObject = JSON.parse(fs.readFileSync(report, 'utf-8')); const reportObject = JSON.parse(fs.readFileSync(report, 'utf-8'));
delete reportObject[0].suites[0].duration; delete reportObject[0].suites[0].duration;
@ -138,7 +158,73 @@ test('should not throw when attachment is missing', async ({ runInlineTest }) =>
testInfo.attachments.push({ name: 'screenshot', path: screenshot, contentType: 'image/png' }); testInfo.attachments.push({ name: 'screenshot', path: screenshot, contentType: 'image/png' });
}); });
`, `,
}, { reporter: 'dot,' + kHTMLReporterPath }); }, { reporter: 'dot,html' });
expect(result.exitCode).toBe(0); expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1); expect(result.passed).toBe(1);
}); });
test('should include image diff', async ({ runInlineTest, page, showReport }) => {
const expected = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAMgAAADICAYAAACtWK6eAAAAAXNSR0IArs4c6QAAAhVJREFUeJzt07ERwCAQwLCQ/Xd+FuDcQiFN4MZrZuYDjv7bAfAyg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAiEDVPZBYx6ffy+AAAAAElFTkSuQmCC', 'base64');
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { use: { viewport: { width: 200, height: 200 }} };
`,
'a.test.js-snapshots/expected-linux.png': expected,
'a.test.js-snapshots/expected-darwin.png': expected,
'a.test.js-snapshots/expected-win32.png': expected,
'a.test.js': `
const { test } = pwt;
test('fails', async ({ page }, testInfo) => {
await page.setContent('<html>Hello World</html>');
const screenshot = await page.screenshot();
await expect(screenshot).toMatchSnapshot('expected.png');
});
`,
}, { reporter: 'dot,html' });
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
await showReport();
await page.click('text=a.test.js');
await page.click('text=fails');
const imageDiff = page.locator('.test-image-mismatch');
const image = imageDiff.locator('img');
await expect(image).toHaveAttribute('src', /.*png/);
const actualSrc = await image.getAttribute('src');
await imageDiff.locator('text=Expected').click();
const expectedSrc = await image.getAttribute('src');
await imageDiff.locator('text=Diff').click();
const diffSrc = await image.getAttribute('src');
const set = new Set([expectedSrc, actualSrc, diffSrc]);
expect(set.size).toBe(3);
});
test('should include screenshot on failure', async ({ runInlineTest, page, showReport }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
use: {
viewport: { width: 200, height: 200 },
screenshot: 'only-on-failure',
}
};
`,
'a.test.js': `
const { test } = pwt;
test('fails', async ({ page }) => {
await page.setContent('<html>Failed state</html>');
await expect(true).toBeFalsy();
});
`,
}, { reporter: 'dot,html' });
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
await showReport();
await page.click('text=a.test.js');
await page.click('text=fails');
await expect(page.locator('text=Screenshots')).toBeVisible();
await expect(page.locator('img')).toBeVisible();
const src = await page.locator('img').getAttribute('src');
expect(src).toBeTruthy();
});