Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

fix(tests): fix a race with golden setup (#1757)

Browse source 
When running multiple browsers, we should take the golden information from the state.
This commit is contained in:
Dmitry Gozman 2020-04-13 14:12:44 -07:00 committed by GitHub
parent 9542f47511
commit 29a6cdff6b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 81 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
test/chromium/coverage.spec.js
View file

@ -135,11 +135,11 @@ describe('CSSCoverage', function() {
{start: 17, end: 38}
]);
});
it('should work with complicated usecases', async function({page, server}) {
it('should work with complicated usecases', async function({page, server, golden}) {
await page.coverage.startCSSCoverage();
await page.goto(server.PREFIX + '/csscoverage/involved.html');
const coverage = await page.coverage.stopCSSCoverage();
expect(JSON.stringify(coverage, null, 2).replace(/:\d{4}\//g, ':<PORT>/')).toBeGolden('csscoverage-involved.txt');
expect(JSON.stringify(coverage, null, 2).replace(/:\d{4}\//g, ':<PORT>/')).toBeGolden(golden('csscoverage-involved.txt'));
});
it('should ignore injected stylesheets', async function({page, server}) {
await page.coverage.startCSSCoverage();

8
test/interception.spec.js
View file

@ -435,7 +435,7 @@ describe('Request.fulfill', function() {
expect(response.statusText()).toBe('Unprocessable Entity');
expect(await page.evaluate(() => document.body.textContent)).toBe('Yo, page!');
});
it('should allow mocking binary responses', async({page, server}) => {
it('should allow mocking binary responses', async({page, server, golden}) => {
await page.route('**/*', route => {
const imageBuffer = fs.readFileSync(path.join(__dirname, 'assets', 'pptr.png'));
route.fulfill({
@ -450,9 +450,9 @@ describe('Request.fulfill', function() {
return new Promise(fulfill => img.onload = fulfill);
}, server.PREFIX);
const img = await page.$('img');
expect(await img.screenshot()).toBeGolden('mock-binary-response.png');
expect(await img.screenshot()).toBeGolden(golden('mock-binary-response.png'));
});
it('should work with file path', async({page, server}) => {
it('should work with file path', async({page, server, golden}) => {
await page.route('**/*', route => route.fulfill({ contentType: 'shouldBeIgnored', path: path.join(__dirname, 'assets', 'pptr.png') }));
await page.evaluate(PREFIX => {
const img = document.createElement('img');
@ -461,7 +461,7 @@ describe('Request.fulfill', function() {
return new Promise(fulfill => img.onload = fulfill);
}, server.PREFIX);
const img = await page.$('img');
expect(await img.screenshot()).toBeGolden('mock-binary-response.png');
expect(await img.screenshot()).toBeGolden(golden('mock-binary-response.png'));
});
it('should stringify intercepted request response headers', async({page, server}) => {
await page.route('**/*', route => {

112
test/screenshot.spec.js
View file

@ -18,13 +18,13 @@
const {FFOX, CHROMIUM, WEBKIT} = require('./utils').testOptions(browserType);
describe('Page.screenshot', function() {
it('should work', async({page, server}) => {
it('should work', async({page, server, golden}) => {
await page.setViewportSize({width: 500, height: 500});
await page.goto(server.PREFIX + '/grid.html');
const screenshot = await page.screenshot();
expect(screenshot).toBeGolden('screenshot-sanity.png');
expect(screenshot).toBeGolden(golden('screenshot-sanity.png'));
});
it('should clip rect', async({page, server}) => {
it('should clip rect', async({page, server, golden}) => {
await page.setViewportSize({width: 500, height: 500});
await page.goto(server.PREFIX + '/grid.html');
const screenshot = await page.screenshot({
@ -35,9 +35,9 @@ describe('Page.screenshot', function() {
height: 100
}
});
expect(screenshot).toBeGolden('screenshot-clip-rect.png');
expect(screenshot).toBeGolden(golden('screenshot-clip-rect.png'));
});
it('should clip rect with fullPage', async({page, server}) => {
it('should clip rect with fullPage', async({page, server, golden}) => {
await page.setViewportSize({width: 500, height: 500});
await page.goto(server.PREFIX + '/grid.html');
await page.evaluate(() => window.scrollBy(150, 200));
@ -50,9 +50,9 @@ describe('Page.screenshot', function() {
height: 100,
},
});
expect(screenshot).toBeGolden('screenshot-clip-rect.png');
expect(screenshot).toBeGolden(golden('screenshot-clip-rect.png'));
});
it('should clip elements to the viewport', async({page, server}) => {
it('should clip elements to the viewport', async({page, server, golden}) => {
await page.setViewportSize({width: 500, height: 500});
await page.goto(server.PREFIX + '/grid.html');
const screenshot = await page.screenshot({
@ -63,7 +63,7 @@ describe('Page.screenshot', function() {
height: 100
}
});
expect(screenshot).toBeGolden('screenshot-offscreen-clip.png');
expect(screenshot).toBeGolden(golden('screenshot-offscreen-clip.png'));
});
it('should throw on clip outside the viewport', async({page, server}) => {
await page.setViewportSize({width: 500, height: 500});
@ -78,7 +78,7 @@ describe('Page.screenshot', function() {
}).catch(error => error);
expect(screenshotError.message).toBe('Clipped area is either empty or outside the resulting image');
});
it('should run in parallel', async({page, server}) => {
it('should run in parallel', async({page, server, golden}) => {
await page.setViewportSize({width: 500, height: 500});
await page.goto(server.PREFIX + '/grid.html');
const promises = [];
@ -93,15 +93,15 @@ describe('Page.screenshot', function() {
}));
}
const screenshots = await Promise.all(promises);
expect(screenshots[1]).toBeGolden('grid-cell-1.png');
expect(screenshots[1]).toBeGolden(golden('grid-cell-1.png'));
});
it('should take fullPage screenshots', async({page, server}) => {
it('should take fullPage screenshots', async({page, server, golden}) => {
await page.setViewportSize({width: 500, height: 500});
await page.goto(server.PREFIX + '/grid.html');
const screenshot = await page.screenshot({
fullPage: true
});
expect(screenshot).toBeGolden('screenshot-grid-fullpage.png');
expect(screenshot).toBeGolden(golden('screenshot-grid-fullpage.png'));
});
it('should restore viewport after fullPage screenshot', async({page, server}) => {
await page.setViewportSize({width: 500, height: 500});
@ -111,7 +111,7 @@ describe('Page.screenshot', function() {
expect(page.viewportSize().width).toBe(500);
expect(page.viewportSize().height).toBe(500);
});
it('should run in parallel in multiple pages', async({page, server, context}) => {
it('should run in parallel in multiple pages', async({page, server, context, golden}) => {
const N = 2;
const pages = await Promise.all(Array(N).fill(0).map(async() => {
const page = await context.newPage();
@ -123,10 +123,10 @@ describe('Page.screenshot', function() {
promises.push(pages[i].screenshot({ clip: { x: 50 * i, y: 0, width: 50, height: 50 } }));
const screenshots = await Promise.all(promises);
for (let i = 0; i < N; ++i)
expect(screenshots[i]).toBeGolden(`grid-cell-${i}.png`);
expect(screenshots[i]).toBeGolden(golden(`grid-cell-${i}.png`));
await Promise.all(pages.map(page => page.close()));
});
it.fail(FFOX)('should allow transparency', async({page, server}) => {
it.fail(FFOX)('should allow transparency', async({page, golden}) => {
await page.setViewportSize({ width: 50, height: 150 });
await page.setContent(`
<style>
@ -138,15 +138,15 @@ describe('Page.screenshot', function() {
<div style="background:transparent"></div>
`);
const screenshot = await page.screenshot({omitBackground: true});
expect(screenshot).toBeGolden('transparent.png');
expect(screenshot).toBeGolden(golden('transparent.png'));
});
it('should render white background on jpeg file', async({page, server}) => {
it('should render white background on jpeg file', async({page, server, golden}) => {
await page.setViewportSize({ width: 100, height: 100 });
await page.goto(server.EMPTY_PAGE);
const screenshot = await page.screenshot({omitBackground: true, type: 'jpeg'});
expect(screenshot).toBeGolden('white.jpg');
expect(screenshot).toBeGolden(golden('white.jpg'));
});
it('should work with odd clip size on Retina displays', async({page, server}) => {
it('should work with odd clip size on Retina displays', async({page, golden}) => {
const screenshot = await page.screenshot({
clip: {
x: 0,
@ -155,49 +155,49 @@ describe('Page.screenshot', function() {
height: 11,
}
});
expect(screenshot).toBeGolden('screenshot-clip-odd-size.png');
expect(screenshot).toBeGolden(golden('screenshot-clip-odd-size.png'));
});
it.skip(FFOX)('should work with a mobile viewport', async({browser, server}) => {
it.skip(FFOX)('should work with a mobile viewport', async({browser, server, golden}) => {
const context = await browser.newContext({ viewport: { width: 320, height: 480 }, isMobile: true });
const page = await context.newPage();
await page.goto(server.PREFIX + '/overflow.html');
const screenshot = await page.screenshot();
expect(screenshot).toBeGolden('screenshot-mobile.png');
expect(screenshot).toBeGolden(golden('screenshot-mobile.png'));
await context.close();
});
it.skip(FFOX)('should work with a mobile viewport and clip', async({browser, server}) => {
it.skip(FFOX)('should work with a mobile viewport and clip', async({browser, server, golden}) => {
const context = await browser.newContext({viewport: { width: 320, height: 480 }, isMobile: true});
const page = await context.newPage();
await page.goto(server.PREFIX + '/overflow.html');
const screenshot = await page.screenshot({ clip: { x: 10, y: 10, width: 100, height: 150 } });
expect(screenshot).toBeGolden('screenshot-mobile-clip.png');
expect(screenshot).toBeGolden(golden('screenshot-mobile-clip.png'));
await context.close();
});
it.skip(FFOX)('should work with a mobile viewport and fullPage', async({browser, server}) => {
it.skip(FFOX)('should work with a mobile viewport and fullPage', async({browser, server, golden}) => {
const context = await browser.newContext({viewport: { width: 320, height: 480 }, isMobile: true});
const page = await context.newPage();
await page.goto(server.PREFIX + '/overflow-large.html');
const screenshot = await page.screenshot({ fullPage: true });
expect(screenshot).toBeGolden('screenshot-mobile-fullpage.png');
expect(screenshot).toBeGolden(golden('screenshot-mobile-fullpage.png'));
await context.close();
});
it('should work for canvas', async({page, server}) => {
it('should work for canvas', async({page, server, golden}) => {
await page.setViewportSize({width: 500, height: 500});
await page.goto(server.PREFIX + '/screenshots/canvas.html');
const screenshot = await page.screenshot();
expect(screenshot).toBeGolden('screenshot-canvas.png');
expect(screenshot).toBeGolden(golden('screenshot-canvas.png'));
});
it('should work for translateZ', async({page, server}) => {
it('should work for translateZ', async({page, server, golden}) => {
await page.setViewportSize({width: 500, height: 500});
await page.goto(server.PREFIX + '/screenshots/translateZ.html');
const screenshot = await page.screenshot();
expect(screenshot).toBeGolden('screenshot-translateZ.png');
expect(screenshot).toBeGolden(golden('screenshot-translateZ.png'));
});
it.fail(FFOX || WEBKIT)('should work for webgl', async({page, server}) => {
it.fail(FFOX || WEBKIT)('should work for webgl', async({page, server, golden}) => {
await page.setViewportSize({width: 640, height: 480});
await page.goto(server.PREFIX + '/screenshots/webgl.html');
const screenshot = await page.screenshot();
expect(screenshot).toBeGolden('screenshot-webgl.png');
expect(screenshot).toBeGolden(golden('screenshot-webgl.png'));
});
it('should work while navigating', async({page, server}) => {
await page.setViewportSize({width: 500, height: 500});
@ -211,26 +211,26 @@ describe('Page.screenshot', function() {
expect(screenshot).toBeInstanceOf(Buffer);
}
});
it('should work with device scale factor', async({browser, server}) => {
it('should work with device scale factor', async({browser, server, golden}) => {
const context = await browser.newContext({ viewport: { width: 320, height: 480 }, deviceScaleFactor: 2 });
const page = await context.newPage();
await page.goto(server.PREFIX + '/grid.html');
const screenshot = await page.screenshot();
expect(screenshot).toBeGolden('screenshot-device-scale-factor.png');
expect(screenshot).toBeGolden(golden('screenshot-device-scale-factor.png'));
await context.close();
});
});
describe('ElementHandle.screenshot', function() {
it('should work', async({page, server}) => {
it('should work', async({page, server, golden}) => {
await page.setViewportSize({width: 500, height: 500});
await page.goto(server.PREFIX + '/grid.html');
await page.evaluate(() => window.scrollBy(50, 100));
const elementHandle = await page.$('.box:nth-of-type(3)');
const screenshot = await elementHandle.screenshot();
expect(screenshot).toBeGolden('screenshot-element-bounding-box.png');
expect(screenshot).toBeGolden(golden('screenshot-element-bounding-box.png'));
});
it('should take into account padding and border', async({page, server}) => {
it('should take into account padding and border', async({page, golden}) => {
await page.setViewportSize({width: 500, height: 500});
await page.setContent(`
<div style="height: 14px">oooo</div>
@ -245,9 +245,9 @@ describe('ElementHandle.screenshot', function() {
`);
const elementHandle = await page.$('div#d');
const screenshot = await elementHandle.screenshot();
expect(screenshot).toBeGolden('screenshot-element-padding-border.png');
expect(screenshot).toBeGolden(golden('screenshot-element-padding-border.png'));
});
it('should capture full element when larger than viewport in parallel', async({page, server}) => {
it('should capture full element when larger than viewport in parallel', async({page, golden}) => {
await page.setViewportSize({width: 500, height: 500});
await page.setContent(`
@ -270,11 +270,11 @@ describe('ElementHandle.screenshot', function() {
const elementHandles = await page.$$('div.to-screenshot');
const promises = elementHandles.map(handle => handle.screenshot());
const screenshots = await Promise.all(promises);
expect(screenshots[2]).toBeGolden('screenshot-element-larger-than-viewport.png');
expect(screenshots[2]).toBeGolden(golden('screenshot-element-larger-than-viewport.png'));
expect(await page.evaluate(() => ({ w: window.innerWidth, h: window.innerHeight }))).toEqual({ w: 500, h: 500 });
});
it('should capture full element when larger than viewport', async({page, server}) => {
it('should capture full element when larger than viewport', async({page, golden}) => {
await page.setViewportSize({width: 500, height: 500});
await page.setContent(`
@ -296,11 +296,11 @@ describe('ElementHandle.screenshot', function() {
`);
const elementHandle = await page.$('div.to-screenshot');
const screenshot = await elementHandle.screenshot();
expect(screenshot).toBeGolden('screenshot-element-larger-than-viewport.png');
expect(screenshot).toBeGolden(golden('screenshot-element-larger-than-viewport.png'));
expect(await page.evaluate(() => ({ w: window.innerWidth, h: window.innerHeight }))).toEqual({ w: 500, h: 500 });
});
it('should scroll element into view', async({page, server}) => {
it('should scroll element into view', async({page, golden}) => {
await page.setViewportSize({width: 500, height: 500});
await page.setContent(`
<div style="height: 14px">oooo</div>
@ -321,9 +321,9 @@ describe('ElementHandle.screenshot', function() {
`);
const elementHandle = await page.$('div.to-screenshot');
const screenshot = await elementHandle.screenshot();
expect(screenshot).toBeGolden('screenshot-element-scrolled-into-view.png');
expect(screenshot).toBeGolden(golden('screenshot-element-scrolled-into-view.png'));
});
it('should scroll 15000px into view', async({page, server}) => {
it('should scroll 15000px into view', async({page, golden}) => {
await page.setViewportSize({width: 500, height: 500});
await page.setContent(`
<div style="height: 14px">oooo</div>
@ -344,9 +344,9 @@ describe('ElementHandle.screenshot', function() {
`);
const elementHandle = await page.$('div.to-screenshot');
const screenshot = await elementHandle.screenshot();
expect(screenshot).toBeGolden('screenshot-element-scrolled-into-view.png');
expect(screenshot).toBeGolden(golden('screenshot-element-scrolled-into-view.png'));
});
it('should work with a rotated element', async({page, server}) => {
it('should work with a rotated element', async({page, golden}) => {
await page.setViewportSize({width: 500, height: 500});
await page.setContent(`<div style="position:absolute;
top: 100px;
@ -357,7 +357,7 @@ describe('ElementHandle.screenshot', function() {
transform: rotateZ(200deg);">&nbsp;</div>`);
const elementHandle = await page.$('div');
const screenshot = await elementHandle.screenshot();
expect(screenshot).toBeGolden('screenshot-element-rotate.png');
expect(screenshot).toBeGolden(golden('screenshot-element-rotate.png'));
});
it('should fail to screenshot a detached element', async({page, server}) => {
await page.setContent('<h1>remove this</h1>');
@ -372,37 +372,37 @@ describe('ElementHandle.screenshot', function() {
const error = await div.screenshot().catch(e => e);
expect(error.message).toBe('Node has 0 height.');
});
it('should work for an element with fractional dimensions', async({page}) => {
it('should work for an element with fractional dimensions', async({page, golden}) => {
await page.setContent('<div style="width:48.51px;height:19.8px;border:1px solid black;"></div>');
const elementHandle = await page.$('div');
const screenshot = await elementHandle.screenshot();
expect(screenshot).toBeGolden('screenshot-element-fractional.png');
expect(screenshot).toBeGolden(golden('screenshot-element-fractional.png'));
});
it.skip(FFOX)('should work with a mobile viewport', async({browser, server}) => {
it.skip(FFOX)('should work with a mobile viewport', async({browser, server, golden}) => {
const context = await browser.newContext({viewport: { width: 320, height: 480, isMobile: true }});
const page = await context.newPage();
await page.goto(server.PREFIX + '/grid.html');
await page.evaluate(() => window.scrollBy(50, 100));
const elementHandle = await page.$('.box:nth-of-type(3)');
const screenshot = await elementHandle.screenshot();
expect(screenshot).toBeGolden('screenshot-element-mobile.png');
expect(screenshot).toBeGolden(golden('screenshot-element-mobile.png'));
await context.close();
});
it.skip(FFOX)('should work with device scale factor', async({browser, server}) => {
it.skip(FFOX)('should work with device scale factor', async({browser, server, golden}) => {
const context = await browser.newContext({ viewport: { width: 320, height: 480 }, deviceScaleFactor: 2 });
const page = await context.newPage();
await page.goto(server.PREFIX + '/grid.html');
await page.evaluate(() => window.scrollBy(50, 100));
const elementHandle = await page.$('.box:nth-of-type(3)');
const screenshot = await elementHandle.screenshot();
expect(screenshot).toBeGolden('screenshot-element-mobile-dsf.png');
expect(screenshot).toBeGolden(golden('screenshot-element-mobile-dsf.png'));
await context.close();
});
it('should work for an element with an offset', async({page}) => {
it('should work for an element with an offset', async({page, golden}) => {
await page.setContent('<div style="position:absolute; top: 10.3px; left: 20.4px;width:50.3px;height:20.2px;border:1px solid black;"></div>');
const elementHandle = await page.$('div');
const screenshot = await elementHandle.screenshot();
expect(screenshot).toBeGolden('screenshot-element-fractional-offset.png');
expect(screenshot).toBeGolden(golden('screenshot-element-fractional-offset.png'));
});
it('should take screenshots when default viewport is null', async({server, browser}) => {
const context = await browser.newContext({ viewport: null });

9
test/test.config.js
View file

@ -63,12 +63,15 @@ serverEnvironment.beforeEach(async({server, httpsServer}) => {
});
const customEnvironment = new Environment('Golden+CheckContexts');
customEnvironment.beforeAll(async ({browserType}) => {
const { OUTPUT_DIR, GOLDEN_DIR } = require('./utils').testOptions(browserType);
customEnvironment.beforeAll(async state => {
const { OUTPUT_DIR, GOLDEN_DIR } = require('./utils').testOptions(state.browserType);
if (fs.existsSync(OUTPUT_DIR))
rm(OUTPUT_DIR);
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
expect.setupGolden(GOLDEN_DIR, OUTPUT_DIR);
state.golden = goldenName => ({ goldenPath: GOLDEN_DIR, outputPath: OUTPUT_DIR, goldenName });
});
customEnvironment.afterAll(async state => {
delete state.golden;
});
customEnvironment.afterEach(async (state, testRun) => {
if (state.browser && state.browser.contexts().length !== 0) {

2
test/types.d.ts vendored
View file

@ -17,7 +17,7 @@ interface Expect<T> {
toBeLessThanOrEqual(other: number, message?: string): void;
toBeCloseTo(other: number, precision: number, message?: string): void;
toBeGolden(path: string): void;
toBeGolden(golden: {goldenPath: string, outputPath: string, goldenName: string}): void;
not: Expect<T>;
}

9
utils/doclint/check_public_api/test/test.js
View file

@ -61,30 +61,27 @@ runner.run();
async function testLint(state, testRun) {
const dirPath = path.join(__dirname, testRun.test().name());
expect.setupGolden(dirPath);
const mdSources = await Source.readdir(dirPath, '.md');
const tsSources = await Source.readdir(dirPath, '.ts');
const jsSources = await Source.readdir(dirPath, '.js');
const messages = await checkPublicAPI(page, mdSources, jsSources.concat(tsSources));
const errors = messages.map(message => message.text);
expect(errors.join('\n')).toBeGolden('result.txt');
expect(errors.join('\n')).toBeGolden({goldenPath: dirPath, outputPath: dirPath, goldenName: 'result.txt'});
}
async function testMDBuilder(state, testRun) {
const dirPath = path.join(__dirname, testRun.test().name());
expect.setupGolden(dirPath);
const sources = await Source.readdir(dirPath, '.md');
const {documentation} = await mdBuilder(page, sources);
expect(serialize(documentation)).toBeGolden('result.txt');
expect(serialize(documentation)).toBeGolden({goldenPath: dirPath, outputPath: dirPath, goldenName: 'result.txt'});
}
async function testJSBuilder(state, testRun) {
const dirPath = path.join(__dirname, testRun.test().name());
expect.setupGolden(dirPath);
const jsSources = await Source.readdir(dirPath, '.js');
const tsSources = await Source.readdir(dirPath, '.ts');
const {documentation} = await jsBuilder.checkSources(jsSources.concat(tsSources));
expect(serialize(documentation)).toBeGolden('result.txt');
expect(serialize(documentation)).toBeGolden({goldenPath: dirPath, outputPath: dirPath, goldenName: 'result.txt'});
}
/**

9
utils/testrunner/GoldenUtils.js
View file

@ -85,12 +85,13 @@ function compareText(actual, expectedBuffer) {
/**
* @param {?Object} actual
* @param {string} goldenName
* @param {!{goldenPath: string, outputPath: string, goldenName: string}} golden
* @return {!{pass: boolean, message: (undefined|string)}}
*/
function compare(goldenPath, outputPath, actual, goldenName) {
goldenPath = path.normalize(goldenPath);
outputPath = path.normalize(outputPath);
function compare(actual, golden) {
const goldenPath = path.normalize(golden.goldenPath);
const outputPath = path.normalize(golden.outputPath);
const goldenName = golden.goldenName;
const expectedPath = path.join(goldenPath, goldenName);
const actualPath = path.join(outputPath, goldenName);

7
utils/testrunner/Matchers.js
View file

@ -25,9 +25,6 @@ class Matchers {
Object.assign(this._matchers, DefaultMatchers);
Object.assign(this._matchers, customMatchers);
this.expect = this.expect.bind(this);
this.expect.setupGolden = (expected, output = expected) => {
this._matchers.toBeGolden = GoldenUtils.compare.bind(null, expected, output);
};
}
addMatcher(name, matcher) {
@ -228,6 +225,10 @@ const DefaultMatchers = {
message = message || `${received.constructor.name} instanceof ${other.name}`;
return { pass: received instanceof other, message };
},
toBeGolden: function(received, golden) {
return GoldenUtils.compare(received, golden);
},
};
function stringify(value) {

0
test/diffstyle.css → utils/testrunner/diffstyle.css
View file