diff --git a/src/server/frames.ts b/src/server/frames.ts index bf7ecd1ac7..6b1ccaf8bb 100644 --- a/src/server/frames.ts +++ b/src/server/frames.ts @@ -24,7 +24,7 @@ import { Page } from './page'; import * as types from './types'; import { BrowserContext } from './browserContext'; import { Progress, ProgressController } from './progress'; -import { assert, makeWaitForNextTask } from '../utils/utils'; +import { assert, createGuid, makeWaitForNextTask } from '../utils/utils'; import { debugLogger } from '../utils/debugLogger'; import { CallMetadata, SdkObject } from './instrumentation'; import { ElementStateWithoutStable } from './injected/injectedScript'; @@ -415,7 +415,7 @@ export class Frame extends SdkObject { constructor(page: Page, id: string, parentFrame: Frame | null) { super(page); - this.uniqueId = parentFrame ? `frame@${page.uniqueId}/${id}` : page.uniqueId; + this.uniqueId = parentFrame ? `frame@${createGuid()}` : page.uniqueId; this.attribution.frame = this; this._id = id; this._page = page; diff --git a/src/server/snapshot/inMemorySnapshotter.ts b/src/server/snapshot/inMemorySnapshotter.ts index 6ec2a30e5f..6f44e39f67 100644 --- a/src/server/snapshot/inMemorySnapshotter.ts +++ b/src/server/snapshot/inMemorySnapshotter.ts @@ -14,24 +14,20 @@ * limitations under the License. */ -import { EventEmitter } from 'events'; import { HttpServer } from '../../utils/httpServer'; import { BrowserContext } from '../browserContext'; import { helper } from '../helper'; import { Page } from '../page'; -import { ContextResources, FrameSnapshot } from './snapshot'; +import { FrameSnapshot, ResourceSnapshot } from './snapshotTypes'; import { SnapshotRenderer } from './snapshotRenderer'; -import { NetworkResponse, SnapshotServer, SnapshotStorage } from './snapshotServer'; -import { Snapshotter, SnapshotterBlob, SnapshotterDelegate, SnapshotterResource } from './snapshotter'; +import { SnapshotServer } from './snapshotServer'; +import { BaseSnapshotStorage } from './snapshotStorage'; +import { Snapshotter, SnapshotterBlob, SnapshotterDelegate } from './snapshotter'; const kSnapshotInterval = 25; -export class InMemorySnapshotter extends EventEmitter implements SnapshotStorage, SnapshotterDelegate { +export class InMemorySnapshotter extends BaseSnapshotStorage implements SnapshotterDelegate { private _blobs = new Map(); - private _resources = new Map(); - private _frameSnapshots = new Map(); - private _snapshots = new Map(); - private _contextResources: ContextResources = new Map(); private _server: HttpServer; private _snapshotter: Snapshotter; @@ -56,14 +52,14 @@ export class InMemorySnapshotter extends EventEmitter implements SnapshotStorage await this._server.stop(); } - async captureSnapshot(page: Page, snapshotId: string): Promise { - if (this._snapshots.has(snapshotId)) - throw new Error('Duplicate snapshotId: ' + snapshotId); + async captureSnapshot(page: Page, snapshotName: string): Promise { + if (this._frameSnapshots.has(snapshotName)) + throw new Error('Duplicate snapshot name: ' + snapshotName); - this._snapshotter.captureSnapshot(page, snapshotId); + this._snapshotter.captureSnapshot(page, snapshotName); return new Promise(fulfill => { const listener = helper.addEventListener(this, 'snapshot', (renderer: SnapshotRenderer) => { - if (renderer.snapshotId === snapshotId) { + if (renderer.snapshotName === snapshotName) { helper.removeEventListeners([listener]); fulfill(renderer); } @@ -79,41 +75,15 @@ export class InMemorySnapshotter extends EventEmitter implements SnapshotStorage this._blobs.set(blob.sha1, blob.buffer); } - onResource(resource: SnapshotterResource): void { - this._resources.set(resource.resourceId, resource); - let resources = this._contextResources.get(resource.url); - if (!resources) { - resources = []; - this._contextResources.set(resource.url, resources); - } - resources.push({ frameId: resource.frameId, resourceId: resource.resourceId }); + onResourceSnapshot(resource: ResourceSnapshot): void { + this.addResource(resource); } onFrameSnapshot(snapshot: FrameSnapshot): void { - let frameSnapshots = this._frameSnapshots.get(snapshot.frameId); - if (!frameSnapshots) { - frameSnapshots = []; - this._frameSnapshots.set(snapshot.frameId, frameSnapshots); - } - frameSnapshots.push(snapshot); - const renderer = new SnapshotRenderer(new Map(this._contextResources), frameSnapshots, frameSnapshots.length - 1); - this._snapshots.set(snapshot.snapshotId, renderer); - this.emit('snapshot', renderer); + this.addFrameSnapshot(snapshot); } resourceContent(sha1: string): Buffer | undefined { return this._blobs.get(sha1); } - - resourceById(resourceId: string): NetworkResponse | undefined { - return this._resources.get(resourceId)!; - } - - snapshotById(snapshotId: string): SnapshotRenderer | undefined { - return this._snapshots.get(snapshotId); - } - - frameSnapshots(frameId: string): FrameSnapshot[] { - return this._frameSnapshots.get(frameId) || []; - } } diff --git a/src/server/snapshot/persistentSnapshotter.ts b/src/server/snapshot/persistentSnapshotter.ts new file mode 100644 index 0000000000..887c71ab82 --- /dev/null +++ b/src/server/snapshot/persistentSnapshotter.ts @@ -0,0 +1,80 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EventEmitter } from 'events'; +import fs from 'fs'; +import path from 'path'; +import util from 'util'; +import { BrowserContext } from '../browserContext'; +import { Page } from '../page'; +import { FrameSnapshot, ResourceSnapshot } from './snapshotTypes'; +import { Snapshotter, SnapshotterBlob, SnapshotterDelegate } from './snapshotter'; + + +const fsWriteFileAsync = util.promisify(fs.writeFile.bind(fs)); +const fsAppendFileAsync = util.promisify(fs.appendFile.bind(fs)); +const fsMkdirAsync = util.promisify(fs.mkdir.bind(fs)); + +const kSnapshotInterval = 100; + +export class PersistentSnapshotter extends EventEmitter implements SnapshotterDelegate { + private _snapshotter: Snapshotter; + private _resourcesDir: string; + private _writeArtifactChain = Promise.resolve(); + private _networkTrace: string; + private _snapshotTrace: string; + + constructor(context: BrowserContext, tracePrefix: string, resourcesDir: string) { + super(); + this._resourcesDir = resourcesDir; + this._networkTrace = tracePrefix + '-network.trace'; + this._snapshotTrace = tracePrefix + '-dom.trace'; + this._snapshotter = new Snapshotter(context, this); + } + + async start(): Promise { + await fsMkdirAsync(this._resourcesDir, {recursive: true}).catch(() => {}); + await this._snapshotter.initialize(); + await this._snapshotter.setAutoSnapshotInterval(kSnapshotInterval); + } + + async dispose() { + this._snapshotter.dispose(); + await this._writeArtifactChain; + } + + captureSnapshot(page: Page, snapshotName: string) { + this._snapshotter.captureSnapshot(page, snapshotName); + } + + onBlob(blob: SnapshotterBlob): void { + this._writeArtifactChain = this._writeArtifactChain.then(async () => { + await fsWriteFileAsync(path.join(this._resourcesDir, blob.sha1), blob.buffer); + }); + } + + onResourceSnapshot(resource: ResourceSnapshot): void { + this._writeArtifactChain = this._writeArtifactChain.then(async () => { + await fsAppendFileAsync(this._networkTrace, JSON.stringify(resource) + '\n'); + }); + } + + onFrameSnapshot(snapshot: FrameSnapshot): void { + this._writeArtifactChain = this._writeArtifactChain.then(async () => { + await fsAppendFileAsync(this._snapshotTrace, JSON.stringify(snapshot) + '\n'); + }); + } +} diff --git a/src/server/snapshot/snapshotRenderer.ts b/src/server/snapshot/snapshotRenderer.ts index 9c227a9930..9ff0b84f0f 100644 --- a/src/server/snapshot/snapshotRenderer.ts +++ b/src/server/snapshot/snapshotRenderer.ts @@ -14,19 +14,23 @@ * limitations under the License. */ -import { ContextResources, FrameSnapshot, NodeSnapshot, RenderedFrameSnapshot } from './snapshot'; +import { ContextResources, FrameSnapshot, NodeSnapshot, RenderedFrameSnapshot } from './snapshotTypes'; export class SnapshotRenderer { private _snapshots: FrameSnapshot[]; private _index: number; private _contextResources: ContextResources; - readonly snapshotId: string; + readonly snapshotName: string | undefined; constructor(contextResources: ContextResources, snapshots: FrameSnapshot[], index: number) { this._contextResources = contextResources; this._snapshots = snapshots; this._index = index; - this.snapshotId = snapshots[index].snapshotId; + this.snapshotName = snapshots[index].snapshotName; + } + + snapshot(): FrameSnapshot { + return this._snapshots[this._index]; } render(): RenderedFrameSnapshot { @@ -69,7 +73,7 @@ export class SnapshotRenderer { let html = visit(snapshot.html, this._index); if (snapshot.doctype) html = `` + html; - html += ``; + html += ``; const resources: { [key: string]: { resourceId: string, sha1?: string } } = {}; for (const [url, contextResources] of this._contextResources) { @@ -113,7 +117,7 @@ function snapshotNodes(snapshot: FrameSnapshot): NodeSnapshot[] { return (snapshot as any)._nodes; } -export function snapshotScript() { +function snapshotScript() { function applyPlaywrightAttributes(shadowAttribute: string, scrollTopAttribute: string, scrollLeftAttribute: string) { const scrollTops: Element[] = []; const scrollLefts: Element[] = []; @@ -126,17 +130,13 @@ export function snapshotScript() { scrollLefts.push(e); for (const iframe of root.querySelectorAll('iframe')) { - const src = iframe.getAttribute('src') || ''; - if (src.startsWith('data:text/html')) - continue; - // Rewrite iframes to use snapshot url (relative to window.location) - // instead of begin relative to the tag. - const index = location.pathname.lastIndexOf('/'); - if (index === -1) - continue; - const pathname = location.pathname.substring(0, index + 1) + src; - const href = location.href.substring(0, location.href.indexOf(location.pathname)) + pathname; - iframe.setAttribute('src', href); + const src = iframe.getAttribute('src'); + if (!src) { + iframe.setAttribute('src', 'data:text/html,Snapshot is not available'); + } else { + // Append query parameters to inherit ?name= or ?time= values from parent. + iframe.setAttribute('src', window.location.origin + src + window.location.search); + } } for (const element of root.querySelectorAll(`template[${shadowAttribute}]`)) { diff --git a/src/server/snapshot/snapshotServer.ts b/src/server/snapshot/snapshotServer.ts index 69d235513c..a91cf7d173 100644 --- a/src/server/snapshot/snapshotServer.ts +++ b/src/server/snapshot/snapshotServer.ts @@ -16,21 +16,9 @@ import * as http from 'http'; import querystring from 'querystring'; -import { SnapshotRenderer } from './snapshotRenderer'; import { HttpServer } from '../../utils/httpServer'; -import type { RenderedFrameSnapshot } from './snapshot'; - -export type NetworkResponse = { - contentType: string; - responseHeaders: { name: string, value: string }[]; - responseSha1: string; -}; - -export interface SnapshotStorage { - resourceContent(sha1: string): Buffer | undefined; - resourceById(resourceId: string): NetworkResponse | undefined; - snapshotById(snapshotId: string): SnapshotRenderer | undefined; -} +import type { RenderedFrameSnapshot } from './snapshotTypes'; +import { SnapshotStorage } from './snapshotStorage'; export class SnapshotServer { private _snapshotStorage: SnapshotStorage; @@ -38,9 +26,7 @@ export class SnapshotServer { constructor(server: HttpServer, snapshotStorage: SnapshotStorage) { this._snapshotStorage = snapshotStorage; - server.routePath('/snapshot/', this._serveSnapshotRoot.bind(this)); - server.routePath('/snapshot/service-worker.js', this._serveServiceWorker.bind(this)); - server.routePath('/snapshot-data', this._serveSnapshot.bind(this)); + server.routePrefix('/snapshot/', this._serveSnapshot.bind(this)); server.routePrefix('/resources/', this._serveResource.bind(this)); } @@ -91,7 +77,7 @@ export class SnapshotServer { next.src = url; }; window.addEventListener('message', event => { - window.showSnapshot(window.location.href + event.data.snapshotId); + window.showSnapshot(window.location.href + event.data.snapshotUrl); }, false); } @@ -135,24 +121,20 @@ export class SnapshotServer { if (pathname === '/snapshot/service-worker.js' || pathname === '/snapshot/') return fetch(event.request); - let snapshotId: string; + const snapshotUrl = request.mode === 'navigate' ? + request.url : (await self.clients.get(event.clientId))!.url; + if (request.mode === 'navigate') { - snapshotId = pathname.substring('/snapshot/'.length); - } else { - const client = (await self.clients.get(event.clientId))!; - snapshotId = new URL(client.url).pathname.substring('/snapshot/'.length); - } - if (request.mode === 'navigate') { - const htmlResponse = await fetch(`/snapshot-data?snapshotId=${snapshotId}`); + const htmlResponse = await fetch(event.request); const { html, resources }: RenderedFrameSnapshot = await htmlResponse.json(); if (!html) return respondNotAvailable(); - snapshotResources.set(snapshotId, resources); + snapshotResources.set(snapshotUrl, resources); const response = new Response(html, { status: 200, headers: { 'Content-Type': 'text/html' } }); return response; } - const resources = snapshotResources.get(snapshotId)!; + const resources = snapshotResources.get(snapshotUrl)!; const urlWithoutHash = removeHash(request.url); const resource = resources[urlWithoutHash]; if (!resource) @@ -193,11 +175,18 @@ export class SnapshotServer { } private _serveSnapshot(request: http.IncomingMessage, response: http.ServerResponse): boolean { + if (request.url!.endsWith('/snapshot/')) + return this._serveSnapshotRoot(request, response); + if (request.url!.endsWith('/snapshot/service-worker.js')) + return this._serveServiceWorker(request, response); + response.statusCode = 200; response.setHeader('Cache-Control', 'public, max-age=31536000'); response.setHeader('Content-Type', 'application/json'); - const parsed: any = querystring.parse(request.url!.substring(request.url!.indexOf('?') + 1)); - const snapshot = this._snapshotStorage.snapshotById(parsed.snapshotId); + const [ pageId, query ] = request.url!.substring('/snapshot/'.length).split('?'); + const parsed: any = querystring.parse(query); + + const snapshot = parsed.name ? this._snapshotStorage.snapshotByName(pageId, parsed.name) : this._snapshotStorage.snapshotByTime(pageId, parsed.time); const snapshotData: any = snapshot ? snapshot.render() : { html: '' }; response.end(JSON.stringify(snapshotData)); return true; diff --git a/src/server/snapshot/snapshotStorage.ts b/src/server/snapshot/snapshotStorage.ts new file mode 100644 index 0000000000..831fb84556 --- /dev/null +++ b/src/server/snapshot/snapshotStorage.ts @@ -0,0 +1,109 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EventEmitter } from 'events'; +import fs from 'fs'; +import path from 'path'; +import util from 'util'; +import { ContextResources, FrameSnapshot, ResourceSnapshot } from './snapshotTypes'; +import { SnapshotRenderer } from './snapshotRenderer'; + +export interface SnapshotStorage { + resources(): ResourceSnapshot[]; + resourceContent(sha1: string): Buffer | undefined; + resourceById(resourceId: string): ResourceSnapshot | undefined; + snapshotByName(frameId: string, snapshotName: string): SnapshotRenderer | undefined; + snapshotByTime(frameId: string, timestamp: number): SnapshotRenderer | undefined; +} + +export abstract class BaseSnapshotStorage extends EventEmitter implements SnapshotStorage { + protected _resources: ResourceSnapshot[] = []; + protected _resourceMap = new Map(); + protected _frameSnapshots = new Map(); + protected _contextResources: ContextResources = new Map(); + + addResource(resource: ResourceSnapshot): void { + this._resourceMap.set(resource.resourceId, resource); + this._resources.push(resource); + let resources = this._contextResources.get(resource.url); + if (!resources) { + resources = []; + this._contextResources.set(resource.url, resources); + } + resources.push({ frameId: resource.frameId, resourceId: resource.resourceId }); + } + + addFrameSnapshot(snapshot: FrameSnapshot): void { + let frameSnapshots = this._frameSnapshots.get(snapshot.frameId); + if (!frameSnapshots) { + frameSnapshots = { + raw: [], + renderer: [], + }; + this._frameSnapshots.set(snapshot.frameId, frameSnapshots); + } + frameSnapshots.raw.push(snapshot); + const renderer = new SnapshotRenderer(new Map(this._contextResources), frameSnapshots.raw, frameSnapshots.raw.length - 1); + frameSnapshots.renderer.push(renderer); + this.emit('snapshot', renderer); + } + + abstract resourceContent(sha1: string): Buffer | undefined; + + resourceById(resourceId: string): ResourceSnapshot | undefined { + return this._resourceMap.get(resourceId)!; + } + + resources(): ResourceSnapshot[] { + return this._resources.slice(); + } + + snapshotByName(frameId: string, snapshotName: string): SnapshotRenderer | undefined { + return this._frameSnapshots.get(frameId)?.renderer.find(r => r.snapshotName === snapshotName); + } + + snapshotByTime(frameId: string, timestamp: number): SnapshotRenderer | undefined { + let result: SnapshotRenderer | undefined = undefined; + for (const snapshot of this._frameSnapshots.get(frameId)?.renderer.values() || []) { + if (timestamp && snapshot.snapshot().timestamp <= timestamp) + result = snapshot; + } + return result; + } +} + +const fsReadFileAsync = util.promisify(fs.readFile.bind(fs)); + +export class PersistentSnapshotStorage extends BaseSnapshotStorage { + private _resourcesDir: any; + + async load(tracePrefix: string, resourcesDir: string) { + this._resourcesDir = resourcesDir; + const networkTrace = await fsReadFileAsync(tracePrefix + '-network.trace', 'utf8'); + const resources = networkTrace.split('\n').map(line => line.trim()).filter(line => !!line).map(line => JSON.parse(line)) as ResourceSnapshot[]; + resources.forEach(r => this.addResource(r)); + const snapshotTrace = await fsReadFileAsync(path.join(tracePrefix + '-dom.trace'), 'utf8'); + const snapshots = snapshotTrace.split('\n').map(line => line.trim()).filter(line => !!line).map(line => JSON.parse(line)) as FrameSnapshot[]; + snapshots.forEach(s => this.addFrameSnapshot(s)); + } + + resourceContent(sha1: string): Buffer | undefined { + return fs.readFileSync(path.join(this._resourcesDir, sha1)); + } +} diff --git a/src/server/snapshot/snapshot.ts b/src/server/snapshot/snapshotTypes.ts similarity index 80% rename from src/server/snapshot/snapshot.ts rename to src/server/snapshot/snapshotTypes.ts index b18a136ff3..85928f5e7a 100644 --- a/src/server/snapshot/snapshot.ts +++ b/src/server/snapshot/snapshotTypes.ts @@ -14,6 +14,21 @@ * limitations under the License. */ +export type ResourceSnapshot = { + resourceId: string, + pageId: string, + frameId: string, + url: string, + contentType: string, + responseHeaders: { name: string, value: string }[], + requestHeaders: { name: string, value: string }[], + method: string, + status: number, + requestSha1: string, + responseSha1: string, + timestamp: number, +}; + export type NodeSnapshot = // Text node. string | @@ -34,10 +49,11 @@ export type ResourceOverride = { }; export type FrameSnapshot = { - snapshotId: string, + snapshotName?: string, pageId: string, frameId: string, frameUrl: string, + timestamp: number, pageTimestamp: number, collectionTime: number, doctype?: string, diff --git a/src/server/snapshot/snapshotter.ts b/src/server/snapshot/snapshotter.ts index 1ae07ca996..b625dc808f 100644 --- a/src/server/snapshot/snapshotter.ts +++ b/src/server/snapshot/snapshotter.ts @@ -21,22 +21,8 @@ import { helper, RegisteredListener } from '../helper'; import { debugLogger } from '../../utils/debugLogger'; import { Frame } from '../frames'; import { SnapshotData, frameSnapshotStreamer, kSnapshotBinding, kSnapshotStreamer } from './snapshotterInjected'; -import { calculateSha1, createGuid } from '../../utils/utils'; -import { FrameSnapshot } from './snapshot'; - -export type SnapshotterResource = { - resourceId: string, - pageId: string, - frameId: string, - url: string, - contentType: string, - responseHeaders: { name: string, value: string }[], - requestHeaders: { name: string, value: string }[], - method: string, - status: number, - requestSha1: string, - responseSha1: string, -}; +import { calculateSha1, createGuid, monotonicTime } from '../../utils/utils'; +import { FrameSnapshot, ResourceSnapshot } from './snapshotTypes'; export type SnapshotterBlob = { buffer: Buffer, @@ -45,7 +31,7 @@ export type SnapshotterBlob = { export interface SnapshotterDelegate { onBlob(blob: SnapshotterBlob): void; - onResource(resource: SnapshotterResource): void; + onResourceSnapshot(resource: ResourceSnapshot): void; onFrameSnapshot(snapshot: FrameSnapshot): void; } @@ -68,13 +54,14 @@ export class Snapshotter { async initialize() { await this._context.exposeBinding(kSnapshotBinding, false, (source, data: SnapshotData) => { const snapshot: FrameSnapshot = { - snapshotId: data.snapshotId, + snapshotName: data.snapshotName, pageId: source.page.uniqueId, frameId: source.frame.uniqueId, frameUrl: data.url, doctype: data.doctype, html: data.html, viewport: data.viewport, + timestamp: monotonicTime(), pageTimestamp: data.timestamp, collectionTime: data.collectionTime, resourceOverrides: [], @@ -105,9 +92,9 @@ export class Snapshotter { helper.removeEventListeners(this._eventListeners); } - captureSnapshot(page: Page, snapshotId: string) { + captureSnapshot(page: Page, snapshotName?: string) { // This needs to be sync, as in not awaiting for anything before we issue the command. - const expression = `window[${JSON.stringify(kSnapshotStreamer)}].captureSnapshot(${JSON.stringify(snapshotId)})`; + const expression = `window[${JSON.stringify(kSnapshotStreamer)}].captureSnapshot(${JSON.stringify(snapshotName)})`; const snapshotFrame = (frame: Frame) => { const context = frame._existingMainContext(); context?.rawEvaluate(expression).catch(debugExceptionHandler); @@ -170,7 +157,7 @@ export class Snapshotter { const requestHeaders = original.headers(); const body = await response.body().catch(e => debugLogger.log('error', e)); const responseSha1 = body ? calculateSha1(body) : 'none'; - const resource: SnapshotterResource = { + const resource: ResourceSnapshot = { pageId: page.uniqueId, frameId: response.frame().uniqueId, resourceId: 'resource@' + createGuid(), @@ -182,8 +169,9 @@ export class Snapshotter { status, requestSha1, responseSha1, + timestamp: monotonicTime() }; - this._delegate.onResource(resource); + this._delegate.onResourceSnapshot(resource); if (requestBody) this._delegate.onBlob({ sha1: requestSha1, buffer: requestBody }); if (body) diff --git a/src/server/snapshot/snapshotterInjected.ts b/src/server/snapshot/snapshotterInjected.ts index 4124b40946..c26f98679a 100644 --- a/src/server/snapshot/snapshotterInjected.ts +++ b/src/server/snapshot/snapshotterInjected.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { NodeSnapshot } from './snapshot'; +import { NodeSnapshot } from './snapshotTypes'; export type SnapshotData = { doctype?: string, @@ -26,7 +26,7 @@ export type SnapshotData = { }[], viewport: { width: number, height: number }, url: string, - snapshotId: string, + snapshotName?: string, timestamp: number, collectionTime: number, }; @@ -168,29 +168,29 @@ export function frameSnapshotStreamer() { (iframeElement as any)[kSnapshotFrameId] = frameId; } - captureSnapshot(snapshotId: string) { - this._streamSnapshot(snapshotId, true); + captureSnapshot(snapshotName?: string) { + this._streamSnapshot(snapshotName); } setSnapshotInterval(interval: number) { this._interval = interval; if (interval) - this._streamSnapshot(`snapshot@${performance.now()}`, false); + this._streamSnapshot(); } - private _streamSnapshot(snapshotId: string, explicitRequest: boolean) { + private _streamSnapshot(snapshotName?: string) { if (this._timer) { clearTimeout(this._timer); this._timer = undefined; } try { - const snapshot = this._captureSnapshot(snapshotId, explicitRequest); + const snapshot = this._captureSnapshot(snapshotName); if (snapshot) (window as any)[kSnapshotBinding](snapshot); } catch (e) { } if (this._interval) - this._timer = setTimeout(() => this._streamSnapshot(`snapshot@${performance.now()}`, false), this._interval); + this._timer = setTimeout(() => this._streamSnapshot(), this._interval); } private _sanitizeUrl(url: string): string { @@ -240,7 +240,7 @@ export function frameSnapshotStreamer() { } } - private _captureSnapshot(snapshotId: string, explicitRequest: boolean): SnapshotData | undefined { + private _captureSnapshot(snapshotName?: string): SnapshotData | undefined { const timestamp = performance.now(); const snapshotNumber = ++this._lastSnapshotNumber; let nodeCounter = 0; @@ -365,6 +365,19 @@ export function frameSnapshotStreamer() { visitChild(child); } + // Process iframe src attribute before bailing out since it depends on a symbol, not the DOM. + if (nodeName === 'IFRAME' || nodeName === 'FRAME') { + const element = node as Element; + for (let i = 0; i < element.attributes.length; i++) { + const frameId = (element as any)[kSnapshotFrameId]; + const name = 'src'; + const value = frameId ? `/snapshot/${frameId}` : ''; + expectValue(name); + expectValue(value); + attrs[name] = value; + } + } + // We can skip attributes comparison because nothing else has changed, // and mutation observer didn't tell us about the attributes. if (equals && data.attributesCached && !shadowDomNesting) @@ -378,22 +391,19 @@ export function frameSnapshotStreamer() { continue; if (nodeName === 'LINK' && name === 'integrity') continue; + if (nodeName === 'IFRAME' && name === 'src') + continue; let value = element.attributes[i].value; - if (name === 'src' && (nodeName === 'IFRAME' || nodeName === 'FRAME')) { - // TODO: handle srcdoc? - const frameId = (element as any)[kSnapshotFrameId]; - value = frameId || 'data:text/html,Snapshot is not available'; - } else if (name === 'src' && (nodeName === 'IMG')) { + if (name === 'src' && (nodeName === 'IMG')) value = this._sanitizeUrl(value); - } else if (name === 'srcset' && (nodeName === 'IMG')) { + else if (name === 'srcset' && (nodeName === 'IMG')) value = this._sanitizeSrcSet(value); - } else if (name === 'srcset' && (nodeName === 'SOURCE')) { + else if (name === 'srcset' && (nodeName === 'SOURCE')) value = this._sanitizeSrcSet(value); - } else if (name === 'href' && (nodeName === 'LINK')) { + else if (name === 'href' && (nodeName === 'LINK')) value = this._sanitizeUrl(value); - } else if (name.startsWith('on')) { + else if (name.startsWith('on')) value = ''; - } expectValue(name); expectValue(value); attrs[name] = value; @@ -424,7 +434,7 @@ export function frameSnapshotStreamer() { height: Math.max(document.body ? document.body.offsetHeight : 0, document.documentElement ? document.documentElement.offsetHeight : 0), }, url: location.href, - snapshotId, + snapshotName, timestamp, collectionTime: 0, }; @@ -444,7 +454,7 @@ export function frameSnapshotStreamer() { } result.collectionTime = performance.now() - result.timestamp; - if (!explicitRequest && htmlEquals && allOverridesAreRefs) + if (!snapshotName && htmlEquals && allOverridesAreRefs) return undefined; return result; } diff --git a/src/server/supplements/injected/recorder.ts b/src/server/supplements/injected/recorder.ts index 8e5a8badd1..ece9f5e74e 100644 --- a/src/server/supplements/injected/recorder.ts +++ b/src/server/supplements/injected/recorder.ts @@ -54,7 +54,7 @@ export class Recorder { private _actionSelector: string | undefined; private _params: { isUnderTest: boolean; }; private _snapshotIframe: HTMLIFrameElement | undefined; - private _snapshotId: string | undefined; + private _snapshotUrl: string | undefined; private _snapshotBaseUrl: string; constructor(injectedScript: InjectedScript, params: { isUnderTest: boolean, snapshotBaseUrl: string }) { @@ -194,7 +194,7 @@ export class Recorder { return; } - const { mode, actionPoint, actionSelector, snapshotId } = state; + const { mode, actionPoint, actionSelector, snapshotUrl } = state; if (mode !== this._mode) { this._mode = mode; this._clearHighlight(); @@ -223,15 +223,15 @@ export class Recorder { this._updateHighlight(); this._actionSelector = actionSelector; } - if (snapshotId !== this._snapshotId) { - this._snapshotId = snapshotId; + if (snapshotUrl !== this._snapshotUrl) { + this._snapshotUrl = snapshotUrl; const snapshotIframe = this._createSnapshotIframeIfNeeded(); if (snapshotIframe) { - if (!snapshotId) { + if (!snapshotUrl) { snapshotIframe.style.visibility = 'hidden'; } else { snapshotIframe.style.visibility = 'visible'; - snapshotIframe.contentWindow?.postMessage({ snapshotId }, '*'); + snapshotIframe.contentWindow?.postMessage({ snapshotUrl }, '*'); } } } diff --git a/src/server/supplements/recorder/recorderTypes.ts b/src/server/supplements/recorder/recorderTypes.ts index 6170e9836b..3c84726a82 100644 --- a/src/server/supplements/recorder/recorderTypes.ts +++ b/src/server/supplements/recorder/recorderTypes.ts @@ -27,7 +27,7 @@ export type UIState = { mode: Mode; actionPoint?: Point; actionSelector?: string; - snapshotId?: string; + snapshotUrl?: string; }; export type CallLog = { diff --git a/src/server/supplements/recorderSupplement.ts b/src/server/supplements/recorderSupplement.ts index 6e358751cc..097dd360d7 100644 --- a/src/server/supplements/recorderSupplement.ts +++ b/src/server/supplements/recorderSupplement.ts @@ -203,12 +203,12 @@ export class RecorderSupplement { (source: BindingSource, action: actions.Action) => this._generator.commitLastAction()); await this._context.exposeBinding('_playwrightRecorderState', false, source => { - let snapshotId: string | undefined; + let snapshotUrl: string | undefined; let actionSelector: string | undefined; let actionPoint: Point | undefined; if (this._hoveredSnapshot) { - snapshotId = this._hoveredSnapshot.phase + '@' + this._hoveredSnapshot.callLogId; - const metadata = this._allMetadatas.get(this._hoveredSnapshot.callLogId); + const metadata = this._allMetadatas.get(this._hoveredSnapshot.callLogId)!; + snapshotUrl = `${metadata.pageId}?name=${this._hoveredSnapshot.phase}@${this._hoveredSnapshot.callLogId}`; actionPoint = this._hoveredSnapshot.phase === 'in' ? metadata?.point : undefined; } else { for (const [metadata, sdkObject] of this._currentCallsMetadata) { @@ -222,7 +222,7 @@ export class RecorderSupplement { mode: this._mode, actionPoint, actionSelector, - snapshotId, + snapshotUrl, }; return uiState; }); @@ -403,9 +403,9 @@ export class RecorderSupplement { _captureSnapshot(sdkObject: SdkObject, metadata: CallMetadata, phase: 'before' | 'after' | 'in') { if (sdkObject.attribution.page) { - const snapshotId = `${phase}@${metadata.id}`; - this._snapshots.add(snapshotId); - this._snapshotter.captureSnapshot(sdkObject.attribution.page, snapshotId); + const snapshotName = `${phase}@${metadata.id}`; + this._snapshots.add(snapshotName); + this._snapshotter.captureSnapshot(sdkObject.attribution.page, snapshotName); } } diff --git a/src/server/trace/common/traceEvents.ts b/src/server/trace/common/traceEvents.ts index 4c4bacc0cf..b41a44a174 100644 --- a/src/server/trace/common/traceEvents.ts +++ b/src/server/trace/common/traceEvents.ts @@ -15,7 +15,6 @@ */ import { StackFrame } from '../../../common/types'; -import { FrameSnapshot } from '../../snapshot/snapshot'; export type ContextCreatedTraceEvent = { timestamp: number, @@ -34,23 +33,6 @@ export type ContextDestroyedTraceEvent = { contextId: string, }; -export type NetworkResourceTraceEvent = { - timestamp: number, - type: 'resource', - contextId: string, - pageId: string, - frameId: string, - resourceId: string, - url: string, - contentType: string, - responseHeaders: { name: string, value: string }[], - requestHeaders: { name: string, value: string }[], - method: string, - status: number, - requestSha1: string, - responseSha1: string, -}; - export type PageCreatedTraceEvent = { timestamp: number, type: 'page-created', @@ -86,7 +68,7 @@ export type ActionTraceEvent = { endTime: number, logs?: string[], error?: string, - snapshots?: { name: string, snapshotId: string }[], + snapshots?: { title: string, snapshotName: string }[], }; export type DialogOpenedEvent = { @@ -122,25 +104,14 @@ export type LoadEvent = { pageId: string, }; -export type FrameSnapshotTraceEvent = { - timestamp: number, - type: 'snapshot', - contextId: string, - pageId: string, - frameId: string, - snapshot: FrameSnapshot, -}; - export type TraceEvent = ContextCreatedTraceEvent | ContextDestroyedTraceEvent | PageCreatedTraceEvent | PageDestroyedTraceEvent | PageVideoTraceEvent | - NetworkResourceTraceEvent | ActionTraceEvent | DialogOpenedEvent | DialogClosedEvent | NavigationEvent | - LoadEvent | - FrameSnapshotTraceEvent; + LoadEvent; diff --git a/src/server/trace/recorder/tracer.ts b/src/server/trace/recorder/tracer.ts index 2594bfa39e..309bcb6aae 100644 --- a/src/server/trace/recorder/tracer.ts +++ b/src/server/trace/recorder/tracer.ts @@ -14,24 +14,20 @@ * limitations under the License. */ -import { BrowserContext, Video } from '../../browserContext'; -import type { SnapshotterResource as SnapshotterResource, SnapshotterBlob, SnapshotterDelegate } from '../../snapshot/snapshotter'; -import * as trace from '../common/traceEvents'; +import fs from 'fs'; import path from 'path'; import * as util from 'util'; -import fs from 'fs'; import { createGuid, getFromENV, mkdirIfNeeded, monotonicTime } from '../../../utils/utils'; -import { Page } from '../../page'; -import { Snapshotter } from '../../snapshot/snapshotter'; -import { helper, RegisteredListener } from '../../helper'; +import { BrowserContext, Video } from '../../browserContext'; import { Dialog } from '../../dialog'; import { Frame, NavigationEvent } from '../../frames'; +import { helper, RegisteredListener } from '../../helper'; import { CallMetadata, InstrumentationListener, SdkObject } from '../../instrumentation'; -import { FrameSnapshot } from '../../snapshot/snapshot'; +import { Page } from '../../page'; +import { PersistentSnapshotter } from '../../snapshot/persistentSnapshotter'; +import * as trace from '../common/traceEvents'; -const fsWriteFileAsync = util.promisify(fs.writeFile.bind(fs)); const fsAppendFileAsync = util.promisify(fs.appendFile.bind(fs)); -const fsAccessAsync = util.promisify(fs.access.bind(fs)); const envTrace = getFromENV('PW_TRACE_DIR'); export class Tracer implements InstrumentationListener { @@ -42,7 +38,7 @@ export class Tracer implements InstrumentationListener { if (!traceDir) return; const traceStorageDir = path.join(traceDir, 'resources'); - const tracePath = path.join(traceDir, createGuid() + '.trace'); + const tracePath = path.join(traceDir, createGuid()); const contextTracer = new ContextTracer(context, traceStorageDir, tracePath); await contextTracer.start(); this._contextTracers.set(context, contextTracer); @@ -72,28 +68,25 @@ export class Tracer implements InstrumentationListener { const snapshotsSymbol = Symbol('snapshots'); // This is an official way to pass snapshots between onBefore/AfterInputAction and onAfterCall. -function snapshotsForMetadata(metadata: CallMetadata): { name: string, snapshotId: string }[] { +function snapshotsForMetadata(metadata: CallMetadata): { title: string, snapshotName: string }[] { if (!(metadata as any)[snapshotsSymbol]) (metadata as any)[snapshotsSymbol] = []; return (metadata as any)[snapshotsSymbol]; } -class ContextTracer implements SnapshotterDelegate { +class ContextTracer { private _contextId: string; - private _traceStoragePromise: Promise; private _appendEventChain: Promise; - private _writeArtifactChain: Promise; - private _snapshotter: Snapshotter; + private _snapshotter: PersistentSnapshotter; private _eventListeners: RegisteredListener[]; private _disposed = false; private _traceFile: string; - constructor(context: BrowserContext, traceStorageDir: string, traceFile: string) { + constructor(context: BrowserContext, traceStorageDir: string, tracePrefix: string) { + const traceFile = tracePrefix + '-actions.trace'; this._contextId = 'context@' + createGuid(); this._traceFile = traceFile; - this._traceStoragePromise = mkdirIfNeeded(path.join(traceStorageDir, 'sha1')).then(() => traceStorageDir); this._appendEventChain = mkdirIfNeeded(traceFile).then(() => traceFile); - this._writeArtifactChain = Promise.resolve(); const event: trace.ContextCreatedTraceEvent = { timestamp: monotonicTime(), type: 'context-created', @@ -105,59 +98,22 @@ class ContextTracer implements SnapshotterDelegate { debugName: context._options._debugName, }; this._appendTraceEvent(event); - this._snapshotter = new Snapshotter(context, this); + this._snapshotter = new PersistentSnapshotter(context, tracePrefix, traceStorageDir); this._eventListeners = [ helper.addEventListener(context, BrowserContext.Events.Page, this._onPage.bind(this)), ]; } async start() { - await this._snapshotter.initialize(); - await this._snapshotter.setAutoSnapshotInterval(100); - } - - onBlob(blob: SnapshotterBlob): void { - this._writeArtifact(blob.sha1, blob.buffer); - } - - onResource(resource: SnapshotterResource): void { - const event: trace.NetworkResourceTraceEvent = { - timestamp: monotonicTime(), - type: 'resource', - contextId: this._contextId, - pageId: resource.pageId, - frameId: resource.frameId, - resourceId: resource.resourceId, - url: resource.url, - contentType: resource.contentType, - responseHeaders: resource.responseHeaders, - requestHeaders: resource.requestHeaders, - method: resource.method, - status: resource.status, - requestSha1: resource.requestSha1, - responseSha1: resource.responseSha1, - }; - this._appendTraceEvent(event); - } - - onFrameSnapshot(snapshot: FrameSnapshot): void { - const event: trace.FrameSnapshotTraceEvent = { - timestamp: monotonicTime(), - type: 'snapshot', - contextId: this._contextId, - pageId: snapshot.pageId, - frameId: snapshot.frameId, - snapshot: snapshot, - }; - this._appendTraceEvent(event); + await this._snapshotter.start(); } async onActionCheckpoint(name: string, sdkObject: SdkObject, metadata: CallMetadata): Promise { if (!sdkObject.attribution.page) return; - const snapshotId = createGuid(); - snapshotsForMetadata(metadata).push({ name, snapshotId }); - this._snapshotter.captureSnapshot(sdkObject.attribution.page, snapshotId); + const snapshotName = `${name}@${metadata.id}`; + snapshotsForMetadata(metadata).push({ title: name, snapshotName }); + this._snapshotter.captureSnapshot(sdkObject.attribution.page, snapshotName); } async onAfterCall(sdkObject: SdkObject, metadata: CallMetadata): Promise { @@ -285,24 +241,7 @@ class ContextTracer implements SnapshotterDelegate { // Ensure all writes are finished. await this._appendEventChain; - await this._writeArtifactChain; - } - - private _writeArtifact(sha1: string, buffer: Buffer) { - // Save all write promises to wait for them in dispose. - const promise = this._innerWriteArtifact(sha1, buffer); - this._writeArtifactChain = this._writeArtifactChain.then(() => promise); - } - - private async _innerWriteArtifact(sha1: string, buffer: Buffer): Promise { - const traceDirectory = await this._traceStoragePromise; - const filePath = path.join(traceDirectory, sha1); - try { - await fsAccessAsync(filePath); - } catch (e) { - // File does not exist - write it. - await fsWriteFileAsync(filePath, buffer); - } + await this._snapshotter.dispose(); } private _appendTraceEvent(event: any) { diff --git a/src/server/trace/viewer/traceModel.ts b/src/server/trace/viewer/traceModel.ts index 4165f63108..208028fa7b 100644 --- a/src/server/trace/viewer/traceModel.ts +++ b/src/server/trace/viewer/traceModel.ts @@ -16,19 +16,32 @@ import { createGuid } from '../../../utils/utils'; import * as trace from '../common/traceEvents'; -import { SnapshotRenderer } from '../../snapshot/snapshotRenderer'; -import { ContextResources } from '../../snapshot/snapshot'; +import { ContextResources, ResourceSnapshot } from '../../snapshot/snapshotTypes'; +import { SnapshotStorage } from '../../snapshot/snapshotStorage'; export * as trace from '../common/traceEvents'; export class TraceModel { contextEntries = new Map(); pageEntries = new Map(); - resourceById = new Map(); contextResources = new Map(); - appendEvents(events: trace.TraceEvent[]) { + appendEvents(events: trace.TraceEvent[], snapshotStorage: SnapshotStorage) { for (const event of events) this.appendEvent(event); + const actions: ActionEntry[] = []; + for (const context of this.contextEntries.values()) { + for (const page of context.pages) + actions.push(...page.actions); + } + + const resources = snapshotStorage.resources().reverse(); + actions.reverse(); + + for (const action of actions) { + while (resources.length && resources[0].timestamp > action.action.timestamp) + action.resources.push(resources.shift()!); + action.resources.reverse(); + } } appendEvent(event: trace.TraceEvent) { @@ -54,9 +67,7 @@ export class TraceModel { created: event, destroyed: undefined as any, actions: [], - resources: [], interestingEvents: [], - snapshotsByFrameId: {}, }; const contextEntry = this.contextEntries.get(event.contextId)!; this.pageEntries.set(event.pageId, { pageEntry, contextEntry }); @@ -75,19 +86,11 @@ export class TraceModel { const action: ActionEntry = { actionId, action: event, - resources: pageEntry.resources, + resources: [] }; - pageEntry.resources = []; pageEntry.actions.push(action); break; } - case 'resource': { - const { pageEntry } = this.pageEntries.get(event.pageId!)!; - const action = pageEntry.actions[pageEntry.actions.length - 1]; - (action || pageEntry).resources.push(event); - this.appendResource(event); - break; - } case 'dialog-opened': case 'dialog-closed': case 'navigation': @@ -96,40 +99,12 @@ export class TraceModel { pageEntry.interestingEvents.push(event); break; } - case 'snapshot': { - const { pageEntry } = this.pageEntries.get(event.pageId!)!; - let snapshots = pageEntry.snapshotsByFrameId[event.frameId]; - if (!snapshots) { - snapshots = []; - pageEntry.snapshotsByFrameId[event.frameId] = snapshots; - } - snapshots.push(event); - for (const override of event.snapshot.resourceOverrides) { - if (override.ref) { - const refOverride = snapshots[snapshots.length - 1 - override.ref]?.snapshot.resourceOverrides.find(o => o.url === override.url); - override.sha1 = refOverride?.sha1; - delete override.ref; - } - } - break; - } } const contextEntry = this.contextEntries.get(event.contextId)!; contextEntry.startTime = Math.min(contextEntry.startTime, event.timestamp); contextEntry.endTime = Math.max(contextEntry.endTime, event.timestamp); } - appendResource(event: trace.NetworkResourceTraceEvent) { - const contextResources = this.contextResources.get(event.contextId)!; - let responseEvents = contextResources.get(event.url); - if (!responseEvents) { - responseEvents = []; - contextResources.set(event.url, responseEvents); - } - responseEvents.push({ frameId: event.frameId, resourceId: event.resourceId }); - this.resourceById.set(event.resourceId, event); - } - actionById(actionId: string): { context: ContextEntry, page: PageEntry, action: ActionEntry } { const [contextId, pageId, actionIndex] = actionId.split('/'); const context = this.contextEntries.get(contextId)!; @@ -151,27 +126,6 @@ export class TraceModel { } return { contextEntry, pageEntry }; } - - findSnapshotById(pageId: string, frameId: string, snapshotId: string): SnapshotRenderer | undefined { - const { pageEntry, contextEntry } = this.pageEntries.get(pageId)!; - const frameSnapshots = pageEntry.snapshotsByFrameId[frameId]; - for (let index = 0; index < frameSnapshots.length; index++) { - if (frameSnapshots[index].snapshot.snapshotId === snapshotId) - return new SnapshotRenderer(this.contextResources.get(contextEntry.created.contextId)!, frameSnapshots.map(fs => fs.snapshot), index); - } - } - - findSnapshotByTime(pageId: string, frameId: string, timestamp: number): SnapshotRenderer | undefined { - const { pageEntry, contextEntry } = this.pageEntries.get(pageId)!; - const frameSnapshots = pageEntry.snapshotsByFrameId[frameId]; - let snapshotIndex = -1; - for (let index = 0; index < frameSnapshots.length; index++) { - const snapshot = frameSnapshots[index]; - if (timestamp && snapshot.timestamp <= timestamp) - snapshotIndex = index; - } - return snapshotIndex >= 0 ? new SnapshotRenderer(this.contextResources.get(contextEntry.created.contextId)!, frameSnapshots.map(fs => fs.snapshot), snapshotIndex) : undefined; - } } export type ContextEntry = { @@ -190,14 +144,12 @@ export type PageEntry = { destroyed: trace.PageDestroyedTraceEvent; actions: ActionEntry[]; interestingEvents: InterestingPageEvent[]; - resources: trace.NetworkResourceTraceEvent[]; - snapshotsByFrameId: { [key: string]: trace.FrameSnapshotTraceEvent[] }; } export type ActionEntry = { actionId: string; action: trace.ActionTraceEvent; - resources: trace.NetworkResourceTraceEvent[]; + resources: ResourceSnapshot[] }; const kInterestingActions = ['click', 'dblclick', 'hover', 'check', 'uncheck', 'tap', 'fill', 'press', 'type', 'selectOption', 'setInputFiles', 'goto', 'setContent', 'goBack', 'goForward', 'reload']; diff --git a/src/server/trace/viewer/traceViewer.ts b/src/server/trace/viewer/traceViewer.ts index 6ff9974618..e0b0954d59 100644 --- a/src/server/trace/viewer/traceViewer.ts +++ b/src/server/trace/viewer/traceViewer.ts @@ -19,10 +19,10 @@ import path from 'path'; import * as playwright from '../../../..'; import * as util from 'util'; import { TraceModel } from './traceModel'; -import { NetworkResourceTraceEvent, TraceEvent } from '../common/traceEvents'; +import { TraceEvent } from '../common/traceEvents'; import { ServerRouteHandler, HttpServer } from '../../../utils/httpServer'; -import { SnapshotServer, SnapshotStorage } from '../../snapshot/snapshotServer'; -import { SnapshotRenderer } from '../../snapshot/snapshotRenderer'; +import { SnapshotServer } from '../../snapshot/snapshotServer'; +import { PersistentSnapshotStorage } from '../../snapshot/snapshotStorage'; const fsReadFileAsync = util.promisify(fs.readFile.bind(fs)); @@ -31,10 +31,10 @@ type TraceViewerDocument = { model: TraceModel; }; -class TraceViewer implements SnapshotStorage { +class TraceViewer { private _document: TraceViewerDocument | undefined; - async load(traceDir: string) { + async show(traceDir: string) { const resourcesDir = path.join(traceDir, 'resources'); const model = new TraceModel(); this._document = { @@ -42,19 +42,6 @@ class TraceViewer implements SnapshotStorage { resourcesDir, }; - for (const name of fs.readdirSync(traceDir)) { - if (!name.endsWith('.trace')) - continue; - const filePath = path.join(traceDir, name); - const traceContent = await fsReadFileAsync(filePath, 'utf8'); - const events = traceContent.split('\n').map(line => line.trim()).filter(line => !!line).map(line => JSON.parse(line)) as TraceEvent[]; - model.appendEvents(events); - } - } - - async show() { - const browser = await playwright.chromium.launch({ headless: false }); - // Served by TraceServer // - "/tracemodel" - json with trace model. // @@ -70,8 +57,16 @@ class TraceViewer implements SnapshotStorage { // - "/snapshot/service-worker.js" - service worker that intercepts snapshot resources // and translates them into "/resources/". + const actionsTrace = fs.readdirSync(traceDir).find(name => name.endsWith('-actions.trace'))!; + const tracePrefix = path.join(traceDir, actionsTrace.substring(0, actionsTrace.indexOf('-actions.trace'))); const server = new HttpServer(); - new SnapshotServer(server, this); + const snapshotStorage = new PersistentSnapshotStorage(); + await snapshotStorage.load(tracePrefix, resourcesDir); + new SnapshotServer(server, snapshotStorage); + + const traceContent = await fsReadFileAsync(path.join(traceDir, actionsTrace), 'utf8'); + const events = traceContent.split('\n').map(line => line.trim()).filter(line => !!line).map(line => JSON.parse(line)) as TraceEvent[]; + model.appendEvents(events, snapshotStorage); const traceModelHandler: ServerRouteHandler = (request, response) => { response.statusCode = 200; @@ -112,45 +107,15 @@ class TraceViewer implements SnapshotStorage { server.routePrefix('/sha1/', sha1Handler); const urlPrefix = await server.start(); + + const browser = await playwright.chromium.launch({ headless: false }); const uiPage = await browser.newPage({ viewport: null }); uiPage.on('close', () => process.exit(0)); await uiPage.goto(urlPrefix + '/traceviewer/traceViewer/index.html'); } - - resourceById(resourceId: string): NetworkResourceTraceEvent | undefined { - const traceModel = this._document!.model; - return traceModel.resourceById.get(resourceId)!; - } - - snapshotById(snapshotId: string): SnapshotRenderer | undefined { - const traceModel = this._document!.model; - const parsed = parseSnapshotName(snapshotId); - const snapshot = parsed.snapshotId ? traceModel.findSnapshotById(parsed.pageId, parsed.frameId, parsed.snapshotId) : traceModel.findSnapshotByTime(parsed.pageId, parsed.frameId, parsed.timestamp!); - return snapshot; - } - - resourceContent(sha1: string): Buffer | undefined { - return fs.readFileSync(path.join(this._document!.resourcesDir, sha1)); - } } export async function showTraceViewer(traceDir: string) { const traceViewer = new TraceViewer(); - if (traceDir) - await traceViewer.load(traceDir); - await traceViewer.show(); -} - -function parseSnapshotName(pathname: string): { pageId: string, frameId: string, timestamp?: number, snapshotId?: string } { - const parts = pathname.split('/'); - // - pageId//snapshotId// - // - pageId//timestamp// - if (parts.length !== 5 || parts[0] !== 'pageId' || (parts[2] !== 'snapshotId' && parts[2] !== 'timestamp')) - throw new Error(`Unexpected path "${pathname}"`); - return { - pageId: parts[1], - frameId: parts[4] === 'main' ? parts[1] : parts[4], - snapshotId: (parts[2] === 'snapshotId' ? parts[3] : undefined), - timestamp: (parts[2] === 'timestamp' ? +parts[3] : undefined), - }; + await traceViewer.show(traceDir); } diff --git a/src/web/traceViewer/ui/networkResourceDetails.tsx b/src/web/traceViewer/ui/networkResourceDetails.tsx index a13b9c988a..f3eee7b020 100644 --- a/src/web/traceViewer/ui/networkResourceDetails.tsx +++ b/src/web/traceViewer/ui/networkResourceDetails.tsx @@ -17,19 +17,19 @@ import './networkResourceDetails.css'; import * as React from 'react'; import { Expandable } from './helpers'; -import { NetworkResourceTraceEvent } from '../../../server/trace/common/traceEvents'; +import type { ResourceSnapshot } from '../../../server/snapshot/snapshotTypes'; const utf8Encoder = new TextDecoder('utf-8'); export const NetworkResourceDetails: React.FunctionComponent<{ - resource: NetworkResourceTraceEvent, + resource: ResourceSnapshot, index: number, selected: boolean, setSelected: React.Dispatch>, }> = ({ resource, index, selected, setSelected }) => { const [expanded, setExpanded] = React.useState(false); const [requestBody, setRequestBody] = React.useState(null); - const [responseBody, setResponseBody] = React.useState(null); + const [responseBody, setResponseBody] = React.useState<{ dataUrl?: string, text?: string } | null>(null); React.useEffect(() => { setExpanded(false); @@ -45,14 +45,22 @@ export const NetworkResourceDetails: React.FunctionComponent<{ } if (resource.responseSha1 !== 'none') { + const useBase64 = resource.contentType.includes('image'); const response = await fetch(`/sha1/${resource.responseSha1}`); - const responseResource = await response.arrayBuffer(); - setResponseBody(responseResource); + if (useBase64) { + const blob = await response.blob(); + const reader = new FileReader(); + const eventPromise = new Promise(f => reader.onload = f); + reader.readAsDataURL(blob); + setResponseBody({ dataUrl: (await eventPromise).target.result }); + } else { + setResponseBody({ text: await response.text() }); + } } }; readResources(); - }, [expanded, resource.responseSha1, resource.requestSha1]); + }, [expanded, resource.responseSha1, resource.requestSha1, resource.contentType]); function formatBody(body: string | null, contentType: string): string { if (body === null) @@ -111,8 +119,8 @@ export const NetworkResourceDetails: React.FunctionComponent<{ {resource.requestSha1 !== 'none' ?
{formatBody(requestBody, requestContentType)}
: ''}

Response Body

{resource.responseSha1 === 'none' ?
Response body is not available for this request.
: ''} - {responseBody !== null && resource.contentType.includes('image') ? : ''} - {responseBody !== null && !resource.contentType.includes('image') ?
{formatBody(utf8Encoder.decode(responseBody), resource.contentType)}
: ''} + {responseBody !== null && responseBody.dataUrl ? : ''} + {responseBody !== null && responseBody.text ?
{formatBody(responseBody.text, resource.contentType)}
: ''} }/> ; diff --git a/src/web/traceViewer/ui/networkTab.tsx b/src/web/traceViewer/ui/networkTab.tsx index 351520cec4..ca33af8077 100644 --- a/src/web/traceViewer/ui/networkTab.tsx +++ b/src/web/traceViewer/ui/networkTab.tsx @@ -30,5 +30,3 @@ export const NetworkTab: React.FunctionComponent<{ }) }; }; - - diff --git a/src/web/traceViewer/ui/snapshotTab.tsx b/src/web/traceViewer/ui/snapshotTab.tsx index 31f4270f07..c30f3bbc72 100644 --- a/src/web/traceViewer/ui/snapshotTab.tsx +++ b/src/web/traceViewer/ui/snapshotTab.tsx @@ -30,14 +30,7 @@ export const SnapshotTab: React.FunctionComponent<{ const [measure, ref] = useMeasure(); const [snapshotIndex, setSnapshotIndex] = React.useState(0); - let snapshots: { name: string, snapshotId?: string, snapshotTime?: number }[] = []; - snapshots = (actionEntry ? (actionEntry.action.snapshots || []) : []).slice(); - if (actionEntry) { - if (!snapshots.length || snapshots[0].name !== 'before') - snapshots.unshift({ name: 'before', snapshotTime: actionEntry ? actionEntry.action.startTime : 0 }); - if (snapshots[snapshots.length - 1].name !== 'after') - snapshots.push({ name: 'after', snapshotTime: actionEntry ? actionEntry.action.endTime : 0 }); - } + const snapshots = actionEntry ? (actionEntry.action.snapshots || []) : []; const { pageId, time } = selection || { pageId: undefined, time: 0 }; const iframeRef = React.createRef(); @@ -45,18 +38,15 @@ export const SnapshotTab: React.FunctionComponent<{ if (!iframeRef.current) return; - // TODO: this logic is copied from SnapshotServer. Find a way to share. - let snapshotUrl = 'data:text/html,Snapshot is not available'; + let snapshotUri = undefined; if (pageId) { - snapshotUrl = `/snapshot/pageId/${pageId}/timestamp/${time}/main`; + snapshotUri = `${pageId}?time=${time}`; } else if (actionEntry) { const snapshot = snapshots[snapshotIndex]; - if (snapshot && snapshot.snapshotTime) - snapshotUrl = `/snapshot/pageId/${actionEntry.action.pageId!}/timestamp/${snapshot.snapshotTime}/main`; - else if (snapshot && snapshot.snapshotId) - snapshotUrl = `/snapshot/pageId/${actionEntry.action.pageId!}/snapshotId/${snapshot.snapshotId}/main`; + if (snapshot && snapshot.snapshotName) + snapshotUri = `${actionEntry.action.pageId}?name=${snapshot.snapshotName}`; } - + const snapshotUrl = snapshotUri ? `${window.location.origin}/snapshot/${snapshotUri}` : 'data:text/html,Snapshot is not available'; try { (iframeRef.current.contentWindow as any).showSnapshot(snapshotUrl); } catch (e) { @@ -71,10 +61,10 @@ export const SnapshotTab: React.FunctionComponent<{ }{!selection && snapshots.map((snapshot, index) => { return
setSnapshotIndex(index)}> - {snapshot.name} + {snapshot.title}
}) } diff --git a/test/assets/snapshot/one.css b/test/assets/snapshot/one.css deleted file mode 100644 index 85a2ba850b..0000000000 --- a/test/assets/snapshot/one.css +++ /dev/null @@ -1,5 +0,0 @@ -@import url(./two.css); - -body { - background-color: pink; -} diff --git a/test/assets/snapshot/snapshot-with-css.html b/test/assets/snapshot/snapshot-with-css.html deleted file mode 100644 index a4f73a52b3..0000000000 --- a/test/assets/snapshot/snapshot-with-css.html +++ /dev/null @@ -1,58 +0,0 @@ - - -
hello, world!
-
- diff --git a/test/assets/snapshot/two.css b/test/assets/snapshot/two.css deleted file mode 100644 index a29db76856..0000000000 --- a/test/assets/snapshot/two.css +++ /dev/null @@ -1,5 +0,0 @@ -.imaged { - width: 200px; - height: 200px; - background: url(../pptr.png); -} diff --git a/test/assets/trace-resources.html b/test/assets/trace-resources.html deleted file mode 100644 index e280a28fe9..0000000000 --- a/test/assets/trace-resources.html +++ /dev/null @@ -1,19 +0,0 @@ - - - - Tracer XHR Network Resource example - - -
-
- - Download - - \ No newline at end of file diff --git a/test/snapshotter.spec.ts b/test/snapshotter.spec.ts index b570855ca3..3f6641a7bd 100644 --- a/test/snapshotter.spec.ts +++ b/test/snapshotter.spec.ts @@ -16,9 +16,12 @@ import { folio as baseFolio } from './fixtures'; import { InMemorySnapshotter } from '../lib/server/snapshot/inMemorySnapshotter'; +import { HttpServer } from '../lib/utils/httpServer'; +import { SnapshotServer } from '../lib/server/snapshot/snapshotServer'; type TestFixtures = { snapshotter: any; + snapshotPort: number; }; export const fixtures = baseFolio.extend(); @@ -29,6 +32,15 @@ fixtures.snapshotter.init(async ({ context, toImpl }, runTest) => { await snapshotter.dispose(); }); +fixtures.snapshotPort.init(async ({ snapshotter, testWorkerIndex }, runTest) => { + const httpServer = new HttpServer(); + new SnapshotServer(httpServer, snapshotter); + const port = 9700 + testWorkerIndex; + httpServer.start(port); + await runTest(port); + httpServer.stop(); +}); + const { it, describe, expect } = fixtures.build(); describe('snapshots', (suite, { mode }) => { @@ -122,12 +134,53 @@ describe('snapshots', (suite, { mode }) => { const { sha1 } = resources[cssHref]; expect(snapshotter.resourceContent(sha1).toString()).toBe('button { color: blue; }'); }); + + it('should capture iframe', (test, { browserName }) => { + test.skip(browserName === 'firefox'); + }, async ({ contextFactory, snapshotter, page, server, snapshotPort, toImpl }) => { + await page.route('**/empty.html', route => { + route.fulfill({ + body: '', + contentType: 'text/html' + }).catch(() => {}); + }); + await page.route('**/iframe.html', route => { + route.fulfill({ + body: '', + contentType: 'text/html' + }).catch(() => {}); + }); + await page.goto(server.EMPTY_PAGE); + + // Marking iframe hierarchy is racy, do not expect snapshot, wait for it. + let counter = 0; + let snapshot: any; + for (; ; ++counter) { + snapshot = await snapshotter.captureSnapshot(toImpl(page), 'snapshot' + counter); + const text = distillSnapshot(snapshot).replace(/frame@[^"]+["]/, '"'); + if (text === '') + break; + await page.waitForTimeout(250); + } + + // Render snapshot, check expectations. + const previewContext = await contextFactory(); + const previewPage = await previewContext.newPage(); + await previewPage.goto(`http://localhost:${snapshotPort}/snapshot/`); + await previewPage.evaluate(snapshotId => { + (window as any).showSnapshot(snapshotId); + }, `${snapshot.snapshot().pageId}?name=snapshot${counter}`); + while (previewPage.frames().length < 4) + await new Promise(f => previewPage.once('frameattached', f)); + const button = await previewPage.frames()[3].waitForSelector('button'); + expect(await button.textContent()).toBe('Hello iframe'); + }); }); function distillSnapshot(snapshot) { const { html } = snapshot.render(); return html - .replace(/