chore: read all traces from the folder (#6134)

src/dispatchers/dispatcher.ts

@@ -206,7 +206,7 @@ export class DispatcherConnection {
       endTime: 0,
       type: dispatcher._type,
       method,
-      params,
+      params: params || {},
       log: [],
     };
 

src/server/browserContext.ts

@@ -16,7 +16,7 @@
  */
 
 import { TimeoutSettings } from '../utils/timeoutSettings';
-import { isDebugMode, mkdirIfNeeded } from '../utils/utils';
+import { isDebugMode, mkdirIfNeeded, createGuid } from '../utils/utils';
 import { Browser, BrowserOptions } from './browser';
 import { Download } from './download';
 import * as frames from './frames';
@@ -380,6 +380,8 @@ export function validateBrowserContextOptions(options: types.BrowserContextOptio
   if (isDebugMode())
     options.bypassCSP = true;
   verifyGeolocation(options.geolocation);
+  if (!options._debugName)
+    options._debugName = createGuid();
 }
 
 export function verifyGeolocation(geolocation?: types.Geolocation) {

src/server/chromium/crPage.ts

@@ -840,7 +840,12 @@ class FrameSession {
   _onScreencastFrame(payload: Protocol.Page.screencastFramePayload) {
     this._client.send('Page.screencastFrameAck', {sessionId: payload.sessionId}).catch(() => {});
     const buffer = Buffer.from(payload.data, 'base64');
-    this._page.emit(Page.Events.ScreencastFrame, { buffer, timestamp: payload.metadata.timestamp });
+    this._page.emit(Page.Events.ScreencastFrame, {
+      buffer,
+      timestamp: payload.metadata.timestamp,
+      width: payload.metadata.deviceWidth,
+      height: payload.metadata.deviceHeight,
+    });
   }
 
   async _createVideoRecorder(screencastId: string, options: types.PageScreencastOptions): Promise<void> {

src/server/snapshot/snapshotStorage.ts

@@ -38,6 +38,13 @@ export abstract class BaseSnapshotStorage extends EventEmitter implements Snapsh
   }>();
   protected _contextResources: ContextResources = new Map();
 
+  clear() {
+    this._resources = [];
+    this._resourceMap.clear();
+    this._frameSnapshots.clear();
+    this._contextResources.clear();
+  }
+
   addResource(resource: ResourceSnapshot): void {
     this._resourceMap.set(resource.resourceId, resource);
     this._resources.push(resource);
@@ -91,10 +98,14 @@ export abstract class BaseSnapshotStorage extends EventEmitter implements Snapsh
 const fsReadFileAsync = util.promisify(fs.readFile.bind(fs));
 
 export class PersistentSnapshotStorage extends BaseSnapshotStorage {
-  private _resourcesDir: any;
+  private _resourcesDir: string;
 
-  async load(tracePrefix: string, resourcesDir: string) {
+  constructor(resourcesDir: string) {
+    super();
     this._resourcesDir = resourcesDir;
+  }
+
+  async load(tracePrefix: string) {
     const networkTrace = await fsReadFileAsync(tracePrefix + '-network.trace', 'utf8');
     const resources = networkTrace.split('\n').map(line => line.trim()).filter(line => !!line).map(line => JSON.parse(line)) as ResourceSnapshot[];
     resources.forEach(r => this.addResource(r));

src/server/trace/common/traceEvents.ts

@@ -53,7 +53,9 @@ export type ScreencastFrameTraceEvent = {
   contextId: string,
   pageId: string,
   pageTimestamp: number,
-  sha1: string
+  sha1: string,
+  width: number,
+  height: number,
 };
 
 export type ActionTraceEvent = {

src/server/trace/recorder/tracer.ts

@@ -39,7 +39,7 @@ export class Tracer implements InstrumentationListener {
     if (!traceDir)
       return;
     const resourcesDir = envTrace || path.join(traceDir, 'resources');
-    const tracePath = path.join(traceDir, createGuid());
+    const tracePath = path.join(traceDir, context._options._debugName!);
     const contextTracer = new ContextTracer(context, resourcesDir, tracePath);
     await contextTracer.start();
     this._contextTracers.set(context, contextTracer);
@@ -201,6 +201,8 @@ class ContextTracer {
         contextId: this._contextId,
         sha1,
         pageTimestamp: params.timestamp,
+        width: params.width,
+        height: params.height,
         timestamp: monotonicTime()
       };
       this._appendTraceEvent(event);

src/server/trace/viewer/traceModel.ts

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-import { createGuid } from '../../../utils/utils';
 import * as trace from '../common/traceEvents';
 import { ContextResources, ResourceSnapshot } from '../../snapshot/snapshotTypes';
 import { SnapshotStorage } from '../../snapshot/snapshotStorage';
@@ -48,7 +47,6 @@ export class TraceModel {
     switch (event.type) {
       case 'context-created': {
         this.contextEntries.set(event.contextId, {
-          name: event.debugName || createGuid(),
           startTime: Number.MAX_VALUE,
           endTime: Number.MIN_VALUE,
           created: event,
@@ -135,7 +133,6 @@ export class TraceModel {
 }
 
 export type ContextEntry = {
-  name: string;
   startTime: number;
   endTime: number;
   created: trace.ContextCreatedTraceEvent;
@@ -150,7 +147,12 @@ export type PageEntry = {
   destroyed: trace.PageDestroyedTraceEvent;
   actions: ActionEntry[];
   interestingEvents: InterestingPageEvent[];
-  screencastFrames: { sha1: string, timestamp: number }[]
+  screencastFrames: {
+    sha1: string,
+    timestamp: number,
+    width: number,
+    height: number,
+  }[]
 }
 
 export type ActionEntry = trace.ActionTraceEvent & {

src/server/trace/viewer/traceViewer.ts

@@ -30,22 +30,12 @@ import { ProgressController } from '../../progress';
 
 const fsReadFileAsync = util.promisify(fs.readFile.bind(fs));
 
-type TraceViewerDocument = {
-  resourcesDir: string;
-  model: TraceModel;
-};
-
 class TraceViewer {
-  private _document: TraceViewerDocument | undefined;
+  private _server: HttpServer;
 
-  async show(traceDir: string, resourcesDir?: string) {
+  constructor(traceDir: string, resourcesDir?: string) {
     if (!resourcesDir)
       resourcesDir = path.join(traceDir, 'resources');
-    const model = new TraceModel();
-    this._document = {
-      model,
-      resourcesDir,
-    };
 
     // Served by TraceServer
     // - "/tracemodel" - json with trace model.
@@ -61,31 +51,48 @@ class TraceViewer {
     // - "/snapshot/pageId/..." - actual snapshot html.
     // - "/snapshot/service-worker.js" - service worker that intercepts snapshot resources
     //   and translates them into "/resources/<resourceId>".
-    const actionsTrace = fs.readdirSync(traceDir).find(name => name.endsWith('-actions.trace'))!;
+    const actionTraces = fs.readdirSync(traceDir).filter(name => name.endsWith('-actions.trace'));
-    const tracePrefix = path.join(traceDir, actionsTrace.substring(0, actionsTrace.indexOf('-actions.trace')));
+    const debugNames = actionTraces.map(name => {
-    const server = new HttpServer();
+      const tracePrefix = path.join(traceDir, name.substring(0, name.indexOf('-actions.trace')));
-    const snapshotStorage = new PersistentSnapshotStorage();
+      return path.basename(tracePrefix);
-    await snapshotStorage.load(tracePrefix, resourcesDir);
+    });
-    new SnapshotServer(server, snapshotStorage);
 
-    const traceContent = await fsReadFileAsync(path.join(traceDir, actionsTrace), 'utf8');
+    this._server = new HttpServer();
-    const events = traceContent.split('\n').map(line => line.trim()).filter(line => !!line).map(line => JSON.parse(line)) as TraceEvent[];
-    model.appendEvents(events, snapshotStorage);
 
-    const traceModelHandler: ServerRouteHandler = (request, response) => {
+    const traceListHandler: ServerRouteHandler = (request, response) => {
       response.statusCode = 200;
       response.setHeader('Content-Type', 'application/json');
-      response.end(JSON.stringify(Array.from(this._document!.model.contextEntries.values())));
+      response.end(JSON.stringify(debugNames));
       return true;
     };
-    server.routePath('/contexts', traceModelHandler);
+    this._server.routePath('/contexts', traceListHandler);
+    const snapshotStorage = new PersistentSnapshotStorage(resourcesDir);
+    new SnapshotServer(this._server, snapshotStorage);
+
+    const traceModelHandler: ServerRouteHandler = (request, response) => {
+      const debugName = request.url!.substring('/context/'.length);
+      const tracePrefix = path.join(traceDir, debugName);
+      snapshotStorage.clear();
+      response.statusCode = 200;
+      response.setHeader('Content-Type', 'application/json');
+      (async () => {
+        await snapshotStorage.load(tracePrefix);
+        const traceContent = await fsReadFileAsync(tracePrefix + '-actions.trace', 'utf8');
+        const events = traceContent.split('\n').map(line => line.trim()).filter(line => !!line).map(line => JSON.parse(line)) as TraceEvent[];
+        const model = new TraceModel();
+        model.appendEvents(events, snapshotStorage);
+        response.end(JSON.stringify(model.contextEntries.values().next().value));
+      })().catch(e => console.error(e));
+      return true;
+    };
+    this._server.routePrefix('/context/', traceModelHandler);
 
     const traceViewerHandler: ServerRouteHandler = (request, response) => {
       const relativePath = request.url!.substring('/traceviewer/'.length);
       const absolutePath = path.join(__dirname, '..', '..', '..', 'web', ...relativePath.split('/'));
-      return server.serveFile(response, absolutePath);
+      return this._server.serveFile(response, absolutePath);
     };
-    server.routePrefix('/traceviewer/', traceViewerHandler);
+    this._server.routePrefix('/traceviewer/', traceViewerHandler);
 
     const fileHandler: ServerRouteHandler = (request, response) => {
       try {
@@ -93,24 +100,24 @@ class TraceViewer {
         const search = url.search;
         if (search[0] !== '?')
           return false;
-        return server.serveFile(response, search.substring(1));
+        return this._server.serveFile(response, search.substring(1));
       } catch (e) {
         return false;
       }
     };
-    server.routePath('/file', fileHandler);
+    this._server.routePath('/file', fileHandler);
 
     const sha1Handler: ServerRouteHandler = (request, response) => {
-      if (!this._document)
-        return false;
       const sha1 = request.url!.substring('/sha1/'.length);
       if (sha1.includes('/'))
         return false;
-      return server.serveFile(response, path.join(this._document.resourcesDir, sha1));
+      return this._server.serveFile(response, path.join(resourcesDir!, sha1));
     };
-    server.routePrefix('/sha1/', sha1Handler);
+    this._server.routePrefix('/sha1/', sha1Handler);
+  }
 
-    const urlPrefix = await server.start();
+  async show() {
+    const urlPrefix = await this._server.start();
 
     const traceViewerPlaywright = createPlaywright(true);
     const args = [
@@ -127,6 +134,7 @@ class TraceViewer {
       headless: !!process.env.PWCLI_HEADLESS_FOR_TEST,
       useWebSocket: isUnderTest()
     });
+
     const controller = new ProgressController(internalCallMetadata(), context._browser);
     await controller.run(async progress => {
       await context._browser._defaultContext!._loadDefaultContextAsIs(progress);
@@ -139,6 +147,6 @@ class TraceViewer {
 }
 
 export async function showTraceViewer(traceDir: string, resourcesDir?: string) {
-  const traceViewer = new TraceViewer();
+  const traceViewer = new TraceViewer(traceDir, resourcesDir);
-  await traceViewer.show(traceDir, resourcesDir);
+  await traceViewer.show();
 }

src/web/traceViewer/index.tsx

@@ -23,6 +23,6 @@ import '../common.css';
 
 (async () => {
   applyTheme();
-  const contexts = await fetch('/contexts').then(response => response.json());
+  const debugNames = await fetch('/contexts').then(response => response.json());
-  ReactDOM.render(<Workbench contexts={contexts} />, document.querySelector('#root'));
+  ReactDOM.render(<Workbench debugNames={debugNames} />, document.querySelector('#root'));
 })();

src/web/traceViewer/ui/contextSelector.tsx

@@ -15,27 +15,26 @@
 */
 
 import * as React from 'react';
-import { ContextEntry } from '../../../server/trace/viewer/traceModel';
 import './contextSelector.css';
 
 export const ContextSelector: React.FunctionComponent<{
-  contexts: ContextEntry[],
+  debugNames: string[],
-  context: ContextEntry,
+  debugName: string,
-  onChange: (contextEntry: ContextEntry) => void,
+  onChange: (debugName: string) => void,
-}> = ({ contexts, context, onChange }) => {
+}> = ({ debugNames, debugName, onChange }) => {
   return (
     <select
       className='context-selector'
       style={{
-        visibility: contexts.length <= 1 ? 'hidden' : 'visible',
+        visibility: debugNames.length <= 1 ? 'hidden' : 'visible',
       }}
-      value={context.created.contextId}
+      value={debugName}
       onChange={e => {
         const newIndex = e.target.selectedIndex;
-        onChange(contexts[newIndex]);
+        onChange(debugNames[newIndex]);
       }}
     >
-      {contexts.map(entry => <option value={entry.created.contextId} key={entry.created.contextId}>{entry.name}</option>)}
+      {debugNames.map(debugName => <option value={debugName} key={debugName}>{debugName}</option>)}
     </select>
   );
 };

src/web/traceViewer/ui/filmStrip.tsx

@@ -18,7 +18,7 @@ import './filmStrip.css';
 import { Boundaries, Size } from '../geometry';
 import * as React from 'react';
 import { useMeasure } from './helpers';
-import { lowerBound } from '../../uiUtils';
+import { upperBound } from '../../uiUtils';
 import { ContextEntry, PageEntry } from '../../../server/trace/viewer/traceModel';
 
 export const FilmStrip: React.FunctionComponent<{
@@ -33,15 +33,13 @@ export const FilmStrip: React.FunctionComponent<{
   let previewImage = undefined;
   if (previewX !== undefined && context.pages.length) {
     const previewTime = boundaries.minimum + (boundaries.maximum - boundaries.minimum) * previewX / measure.width;
-    previewImage = screencastFrames[lowerBound(screencastFrames, previewTime, timeComparator)];
+    previewImage = screencastFrames[upperBound(screencastFrames, previewTime, timeComparator) - 1];
   }
   const previewSize = inscribe(context.created.viewportSize!, { width: 600, height: 600 });
-  console.log(previewSize);
 
   return <div className='film-strip' ref={ref}>{
     context.pages.filter(p => p.screencastFrames.length).map((page, index) => <FilmStripLane
       boundaries={boundaries}
-      viewportSize={context.created.viewportSize!}
       page={page}
       width={measure.width}
       key={index}
@@ -49,12 +47,12 @@ export const FilmStrip: React.FunctionComponent<{
   }
   {previewImage && previewX !== undefined &&
     <div className='film-strip-hover' style={{
-      width: previewSize.width,
+      width: previewImage.width,
-      height: previewSize.height,
+      height: previewImage.height,
       top: measure.bottom + 5,
       left: Math.min(previewX, measure.width - previewSize.width - 10),
     }}>
-      <img src={`/sha1/${previewImage.sha1}`} width={previewSize.width} height={previewSize.height} />
+      <img src={`/sha1/${previewImage.sha1}`} width={previewImage.width} height={previewImage.height} />
     </div>
   }
   </div>;
@@ -62,13 +60,17 @@ export const FilmStrip: React.FunctionComponent<{
 
 const FilmStripLane: React.FunctionComponent<{
   boundaries: Boundaries,
-  viewportSize: Size,
   page: PageEntry,
   width: number,
-}> = ({ boundaries, viewportSize, page, width }) => {
+}> = ({ boundaries, page, width }) => {
+  const viewportSize = { width: 0, height: 0 };
+  const screencastFrames = page.screencastFrames;
+  for (const frame of screencastFrames) {
+    viewportSize.width = Math.max(viewportSize.width, frame.width);
+    viewportSize.height = Math.max(viewportSize.height, frame.height);
+  }
   const frameSize = inscribe(viewportSize!, { width: 200, height: 45 });
   const frameMargin = 2.5;
-  const screencastFrames = page.screencastFrames;
   const startTime = screencastFrames[0].timestamp;
   const endTime = screencastFrames[screencastFrames.length - 1].timestamp;
 
@@ -76,12 +78,13 @@ const FilmStripLane: React.FunctionComponent<{
   const gapLeft = (startTime - boundaries.minimum) / boundariesDuration * width;
   const gapRight = (boundaries.maximum - endTime) / boundariesDuration * width;
   const effectiveWidth = (endTime - startTime) / boundariesDuration * width;
-  const frameCount = effectiveWidth / (frameSize.width + 2 * frameMargin) | 0;
+  const frameCount = effectiveWidth / (frameSize.width + 2 * frameMargin) | 0 + 1;
   const frameDuration = (endTime - startTime) / frameCount;
 
   const frames: JSX.Element[] = [];
-  for (let time = startTime, i = 0; time <= endTime; time += frameDuration, ++i) {
+  let i = 0;
-    const index = lowerBound(screencastFrames, time, timeComparator);
+  for (let time = startTime; time <= endTime; time += frameDuration, ++i) {
+    const index = upperBound(screencastFrames, time, timeComparator) - 1;
     frames.push(<div className='film-strip-frame' key={i} style={{
       width: frameSize.width,
       height: frameSize.height,
@@ -91,6 +94,15 @@ const FilmStripLane: React.FunctionComponent<{
       marginRight: frameMargin,
     }} />);
   }
+  // Always append last frame to show endgame.
+  frames.push(<div className='film-strip-frame' key={i} style={{
+    width: frameSize.width,
+    height: frameSize.height,
+    backgroundImage: `url(/sha1/${screencastFrames[screencastFrames.length - 1].sha1})`,
+    backgroundSize: `${frameSize.width}px ${frameSize.height}px`,
+    margin: frameMargin,
+    marginRight: frameMargin,
+  }} />);
 
   return <div className='film-strip-lane' style={{
     marginLeft: gapLeft + 'px',

src/web/traceViewer/ui/timeline.css

@@ -58,10 +58,6 @@
   left: 0;
 }
 
-.timeline-lane.timeline-labels {
-  margin-top: 10px;
-}
-
 .timeline-lane.timeline-bars {
   pointer-events: auto;
   margin-bottom: 10px;

src/web/traceViewer/ui/timeline.tsx

@@ -54,6 +54,8 @@ export const Timeline: React.FunctionComponent<{
     const bars: TimelineBar[] = [];
     for (const page of context.pages) {
       for (const entry of page.actions) {
+        if (!entry.metadata.params)
+          console.log(entry);
         let detail = entry.metadata.params.selector || '';
         if (entry.metadata.method === 'goto')
           detail = entry.metadata.params.url || '';

src/web/traceViewer/ui/workbench.tsx

@@ -26,15 +26,20 @@ import { SourceTab } from './sourceTab';
 import { SnapshotTab } from './snapshotTab';
 import { LogsTab } from './logsTab';
 import { SplitView } from '../../components/splitView';
+import { useAsyncMemo } from './helpers';
 
 
 export const Workbench: React.FunctionComponent<{
-  contexts: ContextEntry[],
+  debugNames: string[],
-}> = ({ contexts }) => {
+}> = ({ debugNames }) => {
-  const [context, setContext] = React.useState(contexts[0]);
+  const [debugName, setDebugName] = React.useState(debugNames[0]);
   const [selectedAction, setSelectedAction] = React.useState<ActionEntry | undefined>();
   const [highlightedAction, setHighlightedAction] = React.useState<ActionEntry | undefined>();
 
+  let context = useAsyncMemo(async () => {
+    return (await fetch(`/context/${debugName}`).then(response => response.json())) as ContextEntry;
+  }, [debugName], emptyContext);
+
   const actions = React.useMemo(() => {
     const actions: ActionEntry[] = [];
     for (const page of context.pages)
@@ -51,10 +56,10 @@ export const Workbench: React.FunctionComponent<{
       <div className='product'>Playwright</div>
       <div className='spacer'></div>
       <ContextSelector
-        contexts={contexts}
+        debugNames={debugNames}
-        context={context}
+        debugName={debugName}
-        onChange={context => {
+        onChange={debugName => {
-          setContext(context);
+          setDebugName(debugName);
           setSelectedAction(undefined);
         }}
       />
@@ -89,3 +94,25 @@ export const Workbench: React.FunctionComponent<{
     </SplitView>
   </div>;
 };
+
+const now = performance.now();
+const emptyContext: ContextEntry = {
+  startTime: now,
+  endTime: now,
+  created: {
+    timestamp: now,
+    type: 'context-created',
+    browserName: '',
+    contextId: '<empty>',
+    deviceScaleFactor: 1,
+    isMobile: false,
+    viewportSize: { width: 1280, height: 800 },
+    debugName: '<empty>',
+  },
+  destroyed: {
+    timestamp: now,
+    type: 'context-destroyed',
+    contextId: '<empty>',      
+  },
+  pages: []
+};

tests/config/browserEnv.ts

@@ -36,7 +36,7 @@ export type BrowserName = 'chromium' | 'firefox' | 'webkit';
 type TestOptions = {
   mode: 'default' | 'driver' | 'service';
   video?: boolean;
-  trace?: boolean;
+  traceDir?: string;
 };
 
 class DriverMode {
@@ -172,7 +172,7 @@ export class PlaywrightEnv implements Env<PlaywrightTestArgs> {
       testInfo.data.mode = this._options.mode;
     if (this._options.video)
       testInfo.data.video = true;
-    if (this._options.trace)
+    if (this._options.traceDir)
       testInfo.data.trace = true;
     return {
       playwright: this._playwright,
@@ -240,10 +240,11 @@ export class BrowserEnv extends PlaywrightEnv implements Env<BrowserTestArgs> {
 
   async beforeEach(testInfo: TestInfo) {
     const result = await super.beforeEach(testInfo);
-
+    const debugName = path.relative(testInfo.config.outputDir, testInfo.outputPath('')).replace(/[\/\\]/g, '-');
     const contextOptions = {
       recordVideo: this._options.video ? { dir: testInfo.outputPath('') } : undefined,
-      _traceDir: this._options.trace ? testInfo.outputPath('') : undefined,
+      _traceDir: this._options.traceDir,
+      _debugName: debugName,
       ...this._contextOptions,
     } as BrowserContextOptions;
 

tests/config/default.config.ts

@@ -30,7 +30,7 @@ import { CLIEnv } from './cliEnv';
 const config: folio.Config = {
   testDir: path.join(__dirname, '..'),
   outputDir: path.join(__dirname, '..', '..', 'test-results'),
-  timeout: process.env.PWVIDEO ? 60000 : 30000,
+  timeout: process.env.PWVIDEO || process.env.PWTRACE ? 60000 : 30000,
   globalTimeout: 5400000,
 };
 if (process.env.CI) {
@@ -67,7 +67,7 @@ for (const browserName of browsers) {
   const options = {
     mode,
     executablePath,
-    trace: !!process.env.PWTRACE,
+    traceDir: process.env.PWTRACE ? path.join(config.outputDir, 'trace') : undefined,
     headless: !process.env.HEADFUL,
     channel: process.env.PW_CHROMIUM_CHANNEL as any,
     video: !!process.env.PWVIDEO,