diff --git a/package-lock.json b/package-lock.json index 9288bef856..8152bd4a97 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,6 +47,7 @@ "source-map-support": "^0.4.18", "stack-utils": "^2.0.3", "ws": "^7.4.6", + "yauzl": "^2.10.0", "yazl": "^2.5.1" }, "bin": { diff --git a/package.json b/package.json index 653e226a08..10302f0c0f 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "source-map-support": "^0.4.18", "stack-utils": "^2.0.3", "ws": "^7.4.6", + "yauzl": "^2.10.0", "yazl": "^2.5.1" }, "devDependencies": { @@ -137,4 +138,4 @@ "xml2js": "^0.4.23", "yaml": "^1.10.0" } -} \ No newline at end of file +} diff --git a/src/server/snapshot/inMemorySnapshotter.ts b/src/server/snapshot/inMemorySnapshotter.ts index 421ab804c3..9644eb279b 100644 --- a/src/server/snapshot/inMemorySnapshotter.ts +++ b/src/server/snapshot/inMemorySnapshotter.ts @@ -96,7 +96,7 @@ export class InMemorySnapshotter extends BaseSnapshotStorage implements Snapshot this.addFrameSnapshot(snapshot); } - resourceContent(sha1: string): Buffer | undefined { + async resourceContent(sha1: string): Promise { return this._blobs.get(sha1); } } diff --git a/src/server/snapshot/snapshotServer.ts b/src/server/snapshot/snapshotServer.ts index f9e773a2a6..d2ca922dc9 100644 --- a/src/server/snapshot/snapshotServer.ts +++ b/src/server/snapshot/snapshotServer.ts @@ -17,7 +17,7 @@ import * as http from 'http'; import querystring from 'querystring'; import { HttpServer } from '../../utils/httpServer'; -import type { RenderedFrameSnapshot } from './snapshotTypes'; +import type { RenderedFrameSnapshot, ResourceSnapshot } from './snapshotTypes'; import { SnapshotStorage } from './snapshotStorage'; import type { Point } from '../../common/types'; @@ -176,37 +176,41 @@ export class SnapshotServer { const sha1 = resource.response.content._sha1; if (!sha1) return false; + (async () => { + this._innerServeResource(sha1, resource, response); + })().catch(() => {}); + return true; + } - try { - const content = this._snapshotStorage.resourceContent(sha1); - if (!content) - return false; - response.statusCode = 200; - let contentType = resource.response.content.mimeType; - const isTextEncoding = /^text\/|^application\/(javascript|json)/.test(contentType); - if (isTextEncoding && !contentType.includes('charset')) - contentType = `${contentType}; charset=utf-8`; - response.setHeader('Content-Type', contentType); - for (const { name, value } of resource.response.headers) { - try { - response.setHeader(name, value.split('\n')); - } catch (e) { - // Browser is able to handle the header, but Node is not. - // Swallow the error since we cannot do anything meaningful. - } - } - - response.removeHeader('Content-Encoding'); - response.removeHeader('Access-Control-Allow-Origin'); - response.setHeader('Access-Control-Allow-Origin', '*'); - response.removeHeader('Content-Length'); - response.setHeader('Content-Length', content.byteLength); - response.setHeader('Cache-Control', 'public, max-age=31536000'); - response.end(content); - return true; - } catch (e) { - return false; + private async _innerServeResource(sha1: string, resource: ResourceSnapshot, response: http.ServerResponse) { + const content = await this._snapshotStorage.resourceContent(sha1); + if (!content) { + response.statusCode = 404; + response.end(); + return; } + response.statusCode = 200; + let contentType = resource.response.content.mimeType; + const isTextEncoding = /^text\/|^application\/(javascript|json)/.test(contentType); + if (isTextEncoding && !contentType.includes('charset')) + contentType = `${contentType}; charset=utf-8`; + response.setHeader('Content-Type', contentType); + for (const { name, value } of resource.response.headers) { + try { + response.setHeader(name, value.split('\n')); + } catch (e) { + // Browser is able to handle the header, but Node is not. + // Swallow the error since we cannot do anything meaningful. + } + } + + response.removeHeader('Content-Encoding'); + response.removeHeader('Access-Control-Allow-Origin'); + response.setHeader('Access-Control-Allow-Origin', '*'); + response.removeHeader('Content-Length'); + response.setHeader('Content-Length', content.byteLength); + response.setHeader('Cache-Control', 'public, max-age=31536000'); + response.end(content); } } diff --git a/src/server/snapshot/snapshotStorage.ts b/src/server/snapshot/snapshotStorage.ts index 6234ab76d6..5af4cfcd33 100644 --- a/src/server/snapshot/snapshotStorage.ts +++ b/src/server/snapshot/snapshotStorage.ts @@ -20,7 +20,7 @@ import { SnapshotRenderer } from './snapshotRenderer'; export interface SnapshotStorage { resources(): ResourceSnapshot[]; - resourceContent(sha1: string): Buffer | undefined; + resourceContent(sha1: string): Promise; snapshotByName(pageOrFrameId: string, snapshotName: string): SnapshotRenderer | undefined; snapshotByIndex(frameId: string, index: number): SnapshotRenderer | undefined; } @@ -58,7 +58,7 @@ export abstract class BaseSnapshotStorage extends EventEmitter implements Snapsh this.emit('snapshot', renderer); } - abstract resourceContent(sha1: string): Buffer | undefined; + abstract resourceContent(sha1: string): Promise; resources(): ResourceSnapshot[] { return this._resources.slice(); diff --git a/src/server/trace/viewer/traceModel.ts b/src/server/trace/viewer/traceModel.ts index bf960821c0..234f6e19d6 100644 --- a/src/server/trace/viewer/traceModel.ts +++ b/src/server/trace/viewer/traceModel.ts @@ -14,13 +14,12 @@ * limitations under the License. */ -import fs from 'fs'; -import path from 'path'; import * as trace from '../common/traceEvents'; import { ResourceSnapshot } from '../../snapshot/snapshotTypes'; import { BaseSnapshotStorage } from '../../snapshot/snapshotStorage'; import { BrowserContextOptions } from '../../types'; import { shouldCaptureSnapshot, VERSION } from '../recorder/tracing'; +import { VirtualFileSystem } from '../../../utils/vfs'; export * as trace from '../common/traceEvents'; export class TraceModel { @@ -180,14 +179,13 @@ export type PageEntry = { }; export class PersistentSnapshotStorage extends BaseSnapshotStorage { - private _resourcesDir: string; - - constructor(resourcesDir: string) { + private _loader: VirtualFileSystem; + constructor(loader: VirtualFileSystem) { super(); - this._resourcesDir = resourcesDir; + this._loader = loader; } - resourceContent(sha1: string): Buffer | undefined { - return fs.readFileSync(path.join(this._resourcesDir, sha1)); + async resourceContent(sha1: string): Promise { + return this._loader.read('resources/' + sha1); } } diff --git a/src/server/trace/viewer/traceViewer.ts b/src/server/trace/viewer/traceViewer.ts index 03457ec524..230cd58d9b 100644 --- a/src/server/trace/viewer/traceViewer.ts +++ b/src/server/trace/viewer/traceViewer.ts @@ -14,12 +14,12 @@ * limitations under the License. */ -import extract from 'extract-zip'; import fs from 'fs'; import readline from 'readline'; import os from 'os'; import path from 'path'; import rimraf from 'rimraf'; +import stream from 'stream'; import { createPlaywright } from '../../playwright'; import { PersistentSnapshotStorage, TraceModel } from './traceModel'; import { ServerRouteHandler, HttpServer } from '../../../utils/httpServer'; @@ -32,15 +32,20 @@ import { BrowserContext } from '../../browserContext'; import { findChromiumChannel } from '../../../utils/registry'; import { installAppIcon } from '../../chromium/crApp'; import { debugLogger } from '../../../utils/debugLogger'; +import { VirtualFileSystem, RealFileSystem, ZipFileSystem } from '../../../utils/vfs'; export class TraceViewer { + private _vfs: VirtualFileSystem; private _server: HttpServer; private _browserName: string; - constructor(tracesDir: string, browserName: string) { + constructor(vfs: VirtualFileSystem, browserName: string) { + this._vfs = vfs; this._browserName = browserName; - const resourcesDir = path.join(tracesDir, 'resources'); + this._server = new HttpServer(); + } + async init() { // Served by TraceServer // - "/tracemodel" - json with trace model. // @@ -55,14 +60,11 @@ export class TraceViewer { // - "/snapshot/pageId/..." - actual snapshot html. // - "/snapshot/service-worker.js" - service worker that intercepts snapshot resources // and translates them into network requests. - const actionTraces = fs.readdirSync(tracesDir).filter(name => name.endsWith('.trace')); - const debugNames = actionTraces.map(name => { - const tracePrefix = path.join(tracesDir, name.substring(0, name.indexOf('.trace'))); - return path.basename(tracePrefix); + const entries = await this._vfs.entries(); + const debugNames = entries.filter(name => name.endsWith('.trace')).map(name => { + return name.substring(0, name.indexOf('.trace')); }); - this._server = new HttpServer(); - const traceListHandler: ServerRouteHandler = (request, response) => { response.statusCode = 200; response.setHeader('Content-Type', 'application/json'); @@ -70,7 +72,7 @@ export class TraceViewer { return true; }; this._server.routePath('/contexts', traceListHandler); - const snapshotStorage = new PersistentSnapshotStorage(resourcesDir); + const snapshotStorage = new PersistentSnapshotStorage(this._vfs); new SnapshotServer(this._server, snapshotStorage); const traceModelHandler: ServerRouteHandler = (request, response) => { @@ -79,12 +81,12 @@ export class TraceViewer { response.statusCode = 200; response.setHeader('Content-Type', 'application/json'); (async () => { - const traceFile = path.join(tracesDir, debugName + '.trace'); + const traceFile = await this._vfs.readStream(debugName + '.trace'); const match = debugName.match(/^(.*)-\d+$/); - const networkFile = path.join(tracesDir, (match ? match[1] : debugName) + '.network'); + const networkFile = await this._vfs.readStream((match ? match[1] : debugName) + '.network').catch(() => undefined); const model = new TraceModel(snapshotStorage); await appendTraceEvents(model, traceFile); - if (fs.existsSync(networkFile)) + if (networkFile) await appendTraceEvents(model, networkFile); model.build(); response.end(JSON.stringify(model.contextEntry)); @@ -117,7 +119,8 @@ export class TraceViewer { const sha1 = request.url!.substring('/sha1/'.length); if (sha1.includes('/')) return false; - return this._server.serveFile(response, path.join(resourcesDir!, sha1)); + this._server.serveVirtualFile(response, this._vfs, 'resources/' + sha1).catch(() => {}); + return true; }; this._server.routePrefix('/sha1/', sha1Handler); } @@ -163,10 +166,9 @@ export class TraceViewer { } } -async function appendTraceEvents(model: TraceModel, file: string) { - const fileStream = fs.createReadStream(file, 'utf8'); +async function appendTraceEvents(model: TraceModel, input: stream.Readable) { const rl = readline.createInterface({ - input: fileStream, + input, crlfDelay: Infinity }); for await (const line of rl as any) @@ -200,17 +202,12 @@ export async function showTraceViewer(tracePath: string, browserName: string, he } if (stat.isDirectory()) { - const traceViewer = new TraceViewer(tracePath, browserName); + const traceViewer = new TraceViewer(new RealFileSystem(tracePath), browserName); + await traceViewer.init(); return await traceViewer.show(headless); } - const zipFile = tracePath; - try { - await extract(zipFile, { dir }); - } catch (e) { - console.log(`Invalid trace file: ${zipFile}`); // eslint-disable-line no-console - return; - } - const traceViewer = new TraceViewer(dir, browserName); + const traceViewer = new TraceViewer(new ZipFileSystem(tracePath), browserName); + await traceViewer.init(); return await traceViewer.show(headless); } diff --git a/src/utils/httpServer.ts b/src/utils/httpServer.ts index 2380cc82be..ab87f905f2 100644 --- a/src/utils/httpServer.ts +++ b/src/utils/httpServer.ts @@ -20,6 +20,7 @@ import path from 'path'; import { Server as WebSocketServer } from 'ws'; import * as mime from 'mime'; import { assert } from './utils'; +import { VirtualFileSystem } from './vfs'; export type ServerRouteHandler = (request: http.IncomingMessage, response: http.ServerResponse) => boolean; @@ -95,6 +96,22 @@ export class HttpServer { } } + async serveVirtualFile(response: http.ServerResponse, vfs: VirtualFileSystem, entry: string, headers?: { [name: string]: string }) { + try { + const content = await vfs.read(entry); + response.statusCode = 200; + const contentType = mime.getType(path.extname(entry)) || 'application/octet-stream'; + response.setHeader('Content-Type', contentType); + response.setHeader('Content-Length', content.byteLength); + for (const [name, value] of Object.entries(headers || {})) + response.setHeader(name, value); + response.end(content); + return true; + } catch (e) { + return false; + } + } + private _onRequest(request: http.IncomingMessage, response: http.ServerResponse) { request.on('error', () => response.end()); try { diff --git a/src/utils/vfs.ts b/src/utils/vfs.ts new file mode 100644 index 0000000000..4b487c9ecb --- /dev/null +++ b/src/utils/vfs.ts @@ -0,0 +1,124 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import path from 'path'; +import fs from 'fs'; +import stream from 'stream'; +import yauzl from 'yauzl'; + +export interface VirtualFileSystem { + entries(): Promise; + read(entry: string): Promise; + readStream(entryPath: string): Promise; + close(): void; +} + +abstract class BaseFileSystem { + + abstract readStream(entryPath: string): Promise; + + async read(entryPath: string): Promise { + const readStream = await this.readStream(entryPath); + const buffers: Buffer[] = []; + return new Promise(f => { + readStream.on('data', d => buffers.push(d)); + readStream.on('end', () => f(Buffer.concat(buffers))); + }); + } + + close() { + } +} + +export class RealFileSystem extends BaseFileSystem implements VirtualFileSystem { + private _folder: string; + + constructor(folder: string) { + super(); + this._folder = folder; + } + + async entries(): Promise { + const result: string[] = []; + const visit = (dir: string) => { + for (const name of fs.readdirSync(dir)) { + const fqn = path.join(dir, name); + if (fs.statSync(fqn).isDirectory()) + visit(fqn); + if (fs.statSync(fqn).isFile()) + result.push(fqn); + } + }; + visit(this._folder); + return result; + } + + async readStream(entry: string): Promise { + return fs.createReadStream(path.join(this._folder, ...entry.split('/'))); + } +} + +export class ZipFileSystem extends BaseFileSystem implements VirtualFileSystem { + private _fileName: string; + private _zipFile: yauzl.ZipFile | undefined; + private _entries = new Map(); + private _openedPromise: Promise; + + constructor(fileName: string) { + super(); + this._fileName = fileName; + this._openedPromise = this.open(); + } + + async open() { + await new Promise((fulfill, reject) => { + yauzl.open(this._fileName, { autoClose: false }, (e, z) => { + if (e) { + reject(e); + return; + } + this._zipFile = z; + this._zipFile!.on('entry', (entry: yauzl.Entry) => { + this._entries.set(entry.fileName, entry); + }); + this._zipFile!.on('end', fulfill); + }); + }); + } + + async entries(): Promise { + await this._openedPromise; + return [...this._entries.keys()]; + } + + async readStream(entryPath: string): Promise { + await this._openedPromise; + const entry = this._entries.get(entryPath)!; + return new Promise((f, r) => { + this._zipFile!.openReadStream(entry, (error, readStream) => { + if (error || !readStream) { + r(error || 'Entry not found'); + return; + } + f(readStream); + }); + }); + } + + override close() { + this._zipFile?.close(); + } +} diff --git a/tests/snapshotter.spec.ts b/tests/snapshotter.spec.ts index 5ca175d37e..d726d464ce 100644 --- a/tests/snapshotter.spec.ts +++ b/tests/snapshotter.spec.ts @@ -146,7 +146,7 @@ it.describe('snapshots', () => { await page.evaluate(() => { (document.styleSheets[0].cssRules[0] as any).style.color = 'blue'; }); const snapshot2 = await snapshotter.captureSnapshot(toImpl(page), 'snapshot1'); const resource = snapshot2.resourceByUrl(`http://localhost:${server.PORT}/style.css`); - expect(snapshotter.resourceContent(resource.response.content._sha1).toString()).toBe('button { color: blue; }'); + expect((await snapshotter.resourceContent(resource.response.content._sha1)).toString()).toBe('button { color: blue; }'); }); it('should capture iframe', async ({ page, server, toImpl, browserName, snapshotter, showSnapshot }) => { diff --git a/tests/tracing.spec.ts b/tests/tracing.spec.ts index 5d78fdf7ac..7ed91ce2e3 100644 --- a/tests/tracing.spec.ts +++ b/tests/tracing.spec.ts @@ -15,7 +15,7 @@ */ import { expect, contextTest as test, browserTest } from './config/browserTest'; -import yauzl from 'yauzl'; +import { ZipFileSystem } from '../lib/utils/vfs'; import jpeg from 'jpeg-js'; test.skip(({ trace }) => !!trace); @@ -284,29 +284,12 @@ test('should not hang for clicks that open dialogs', async ({ context, page }) = }); async function parseTrace(file: string): Promise<{ events: any[], resources: Map }> { - const entries = await new Promise(f => { - const entries: Promise[] = []; - yauzl.open(file, (err, zipFile) => { - zipFile.on('entry', entry => { - const entryPromise = new Promise(ff => { - zipFile.openReadStream(entry, (err, readStream) => { - const buffers = []; - if (readStream) { - readStream.on('data', d => buffers.push(d)); - readStream.on('end', () => ff({ name: entry.fileName, buffer: Buffer.concat(buffers) })); - } else { - ff({ name: entry.fileName }); - } - }); - }); - entries.push(entryPromise); - }); - zipFile.on('end', () => f(entries)); - }); - }); + const zipFS = new ZipFileSystem(file); const resources = new Map(); - for (const { name, buffer } of await Promise.all(entries)) - resources.set(name, buffer); + for (const entry of await zipFS.entries()) + resources.set(entry, await zipFS.read(entry)); + zipFS.close(); + const events = []; for (const line of resources.get('trace.trace').toString().split('\n')) { if (line)