chore: add k8s grid deployments (#26359)

package-lock.json

@@ -6568,6 +6568,7 @@
       "dependencies": {
         "commander": "^11.0.0",
         "debug": "^4.3.2",
+        "playwright-core": "1.37.0-alpha-aug-7-2023",
         "ws": "^8.1.0"
       },
       "bin": {
@@ -6576,8 +6577,7 @@
       "devDependencies": {
         "@types/commander": "^2.12.2",
         "@types/debug": "^4.1.8",
-        "@types/ws": "^8.5.5",
+        "@types/ws": "^8.5.5"
-        "playwright-core": "1.37.0-next"
       },
       "engines": {
         "node": ">=16"
@@ -6591,6 +6591,17 @@
         "node": ">=16"
       }
     },
+    "packages/playwright-grid/node_modules/playwright-core": {
+      "version": "1.37.0-alpha-aug-7-2023",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.37.0-alpha-aug-7-2023.tgz",
+      "integrity": "sha512-heSES+oWES3ktYWiAwi0Oo+UWKCNIJtJVn8h0cgHd7qT2lZ23Iq8DPBBFHtbv+YLjhmXrXRpMIkk4o1MXguLgg==",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=16"
+      }
+    },
     "packages/playwright-test": {
       "name": "@playwright/test",
       "version": "1.37.0-next",
@@ -7571,7 +7582,7 @@
         "@types/ws": "^8.5.5",
         "commander": "^11.0.0",
         "debug": "^4.3.2",
-        "playwright-core": "1.37.0-next",
+        "playwright-core": "1.37.0-alpha-aug-7-2023",
         "ws": "^8.1.0"
       },
       "dependencies": {
@@ -7579,6 +7590,11 @@
           "version": "11.0.0",
           "resolved": "https://registry.npmjs.org/commander/-/commander-11.0.0.tgz",
           "integrity": "sha512-9HMlXtt/BNoYr8ooyjjNRdIilOTkVJXB+GhxMTtOKwk0R4j4lS4NpjuqmRxroBfnfTSHQIHQB7wryHhXarNjmQ=="
+        },
+        "playwright-core": {
+          "version": "1.37.0-alpha-aug-7-2023",
+          "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.37.0-alpha-aug-7-2023.tgz",
+          "integrity": "sha512-heSES+oWES3ktYWiAwi0Oo+UWKCNIJtJVn8h0cgHd7qT2lZ23Iq8DPBBFHtbv+YLjhmXrXRpMIkk4o1MXguLgg=="
         }
       }
     },

packages/playwright-core/src/server/index.ts

@@ -31,3 +31,4 @@ export type { DispatcherScope } from './dispatchers/dispatcher';
 export type { Playwright } from './playwright';
 export { openTraceInBrowser, openTraceViewerApp } from './trace/viewer/traceViewer';
 export { serverSideCallMetadata } from './instrumentation';
+export { SocksProxy } from '../common/socksProxy';

packages/playwright-grid/Dockerfile

@@ -0,0 +1,8 @@
+FROM mcr.microsoft.com/playwright:v1.37.0-alpha-aug-7-2023-jammy
+
+WORKDIR /app
+
+COPY package.json ./
+COPY cli.js ./
+COPY lib ./lib
+RUN npm install

packages/playwright-grid/deployment-grid.yaml

@@ -0,0 +1,42 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grid-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: grid
+  template:
+    metadata:
+      labels:
+        app: grid
+    spec:
+      containers:
+      - name: grid
+        image: playwright-grid
+        imagePullPolicy: IfNotPresent
+        env:
+        - name: DEBUG
+          value: "pw:grid*"
+        - name: PLAYWRIGHT_GRID_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: access-key-secret
+              key: access-key
+        command: ["node", "./cli.js"]
+        args: ["grid", "--port=3000"]
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: grid-service
+spec:
+  selector:
+    app: grid
+  ports:
+    - protocol: TCP
+      port: 3000
+      targetPort: 3000
+  type: LoadBalancer

packages/playwright-grid/deployment-worker.yaml

@@ -0,0 +1,28 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: worker-deployment
+spec:
+  replicas: 10  # or however many nodes you want
+  selector:
+    matchLabels:
+      app: worker
+  template:
+    metadata:
+      labels:
+        app: worker
+    spec:
+      containers:
+      - name: grid
+        image: playwright-grid
+        imagePullPolicy: IfNotPresent
+        env:
+        - name: DEBUG
+          value: "pw:grid*"
+        - name: PLAYWRIGHT_GRID_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: access-key-secret
+              key: access-key
+        command: ["node", "./cli.js"]
+        args: ["node", "--grid=grid-service:3000"]

packages/playwright-grid/docs/azure.md

@@ -0,0 +1,55 @@
+```sh
+# Create resource group
+az group create --name group-grid-001 --location westus3
+
+# Create ACR
+az acr create --resource-group group-grid-001 --name acrgrid001 --sku Basic
+az acr login --name acrgrid001
+az acr list --resource-group group-grid-001 --query "[].{acrLoginServer:loginServer}" --output table
+
+# Create AKS
+az aks create --resource-group group-grid-001 --name aks-grid-001 --node-count 4 --enable-addons monitoring --generate-ssh-keys
+az aks get-credentials --resource-group group-grid-001 --name aks-grid-001
+
+# Grant AKS access to ACR
+az aks show --resource-group group-grid-001 --name aks-grid-001 --query "servicePrincipalProfile.clientId" --output tsv
+# az aks show --resource-group group-grid-001 --name aks-grid-001 --query "identityProfile.kubeletidentity.clientId" -o tsv
+# az acr show --name acrgrid001 --resource-group group-grid-001 --query "id" -o tsv
+# az role assignment create --assignee <GUID> --role AcrPull --scope <SCOP PATH>
+
+# Create secrets
+kubectl create secret generic access-key-secret --from-literal=access-key=$PLAYWRIGHT_GRID_ACCESS_KEY
+
+# Create TLS
+# kubectl create secret tls grid-tls-secret --cert=../../tests/config/testserver/cert.pem --key=../../tests/config/testserver/key.pem
+# az network public-ip create --resource-group MC_group-grid-001_aks-grid-001_westus3 --name public-ip-grid-001 --sku Standard --allocation-method static
+# az network public-ip show --resource-group MC_group-grid-001_aks-grid-001_westus3 --name public-ip-grid-001 --query ipAddress --output tsv
+# # use output below
+# helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
+# helm install nginx-ingress ingress-nginx/ingress-nginx \
+#     --set controller.replicaCount=1 \
+#     --set controller.nodeSelector."beta\.kubernetes\.io/os"=linux \
+#     --set defaultBackend.nodeSelector."beta\.kubernetes\.io/os"=linux \
+#     --set controller.service.loadBalancerIP="20.118.130.255"
+
+# Push Docker container
+docker build -t playwright-grid:latest -f Dockerfile .
+docker tag playwright-grid acrgrid001.azurecr.io/playwright-grid
+docker push acrgrid001.azurecr.io/playwright-grid
+
+# Delete deployment
+kubectl delete deployment grid-deployment
+kubectl delete deployment worker-deployment
+kubectl delete svc grid-service
+
+# Update deployment
+kubectl apply -f deployment-grid.yaml
+kubectl apply -f deployment-worker.yaml
+
+# Debug
+kubectl get pods -l app=grid
+kubectl logs grid-6cbbfc866c-wh8dw
+kubectl get pods -n ingress-basic
+kubectl get svc grid-service
+az aks show --resource-group group-grid-001 --name aks-grid-001 --query fqdn --output tsv
+```

packages/playwright-grid/docs/minikube.md

@@ -0,0 +1,31 @@
+```sh
+minikube config set memory 65536
+minikube config set cpus 12
+minikube start
+minikube dashboard
+
+# Point docker to minikube
+minikube -p minikube docker-env
+eval $(minikube docker-env)
+kubectl config use-context minikube
+kubectl create secret generic access-key-secret --from-literal=access-key=$PLAYWRIGHT_GRID_ACCESS_KEY
+
+# Push Docker container
+docker build -t playwright-grid:latest -f Dockerfile .
+
+# Delete deployment
+kubectl delete deployment grid-deployment
+kubectl delete deployment worker-deployment
+kubectl delete svc grid-service
+
+# Update deployment
+
+kubectl apply -f deployment-grid.yaml
+kubectl apply -f deployment-worker.yaml
+
+# Debug
+minikube ip
+kubectl get svc grid-service
+kubectl get pods -l app=grid
+kubectl logs grid-6cbbfc866c-wh8dw
+```

packages/playwright-grid/package.json

@@ -10,13 +10,13 @@
   "dependencies": {
     "commander": "^11.0.0",
     "debug": "^4.3.2",
+    "playwright-core": "1.37.0-alpha-aug-7-2023",
     "ws": "^8.1.0"
   },
   "devDependencies": {
     "@types/commander": "^2.12.2",
     "@types/debug": "^4.1.8",
-    "@types/ws": "^8.5.5",
+    "@types/ws": "^8.5.5"
-    "playwright-core": "1.37.0-next"
   },
   "repository": "github:Microsoft/playwright",
   "engines": {

packages/playwright-grid/src/cli.ts

@@ -27,7 +27,7 @@ program
     .option('--access-key <key>', 'access key to the grid')
     .action(async opts => {
       const port = opts.port || +(process.env.PLAYWRIGHT_GRID_PORT || '3333');
-      const accessKey = opts.accessKey || process.env.PLAYWRIGHT_GRID_ACCESS_KEY;
+      const accessKey = opts.accessKey || (process.env.PLAYWRIGHT_GRID_ACCESS_KEY || '');
       const { Grid } = await import('./grid/grid.js');
       const grid = new Grid(port, accessKey);
       grid.start();
@@ -40,7 +40,9 @@ program
     .option('--access-key <key>', 'access key to the grid', '')
     .action(async opts => {
       const { Node } = await import('./node/node.js');
-      new Node(opts.grid, +opts.capacity, opts.accessKey);
+      const accessKey = opts.accessKey || (process.env.PLAYWRIGHT_GRID_ACCESS_KEY || '');
+      const node = new Node(opts.grid, +opts.capacity, accessKey);
+      await node.connect();
     });
 
 program.parse(process.argv);

packages/playwright-grid/src/common/httpServer.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import debug from 'debug';
 import fs from 'fs';
 import http from 'http';
 import path from 'path';
@@ -23,11 +24,13 @@ import { Server as WebSocketServer } from 'ws';
 export type ServerRouteHandler = (request: http.IncomingMessage, response: http.ServerResponse) => boolean;
 
 export class HttpServer {
+  private _log: debug.Debugger;
   readonly server: http.Server;
   private _urlPrefix: string;
   private _routes: { prefix?: string, exact?: string, handler: ServerRouteHandler }[] = [];
 
   constructor() {
+    this._log = debug(`pw:grid:http`);
     this._urlPrefix = '';
     this.server = http.createServer(this._onRequest.bind(this));
   }
@@ -45,6 +48,7 @@ export class HttpServer {
   }
 
   async start(port?: number): Promise<string> {
+    this._log('starting server', port);
     this.server.listen(port);
     await new Promise(cb => this.server!.once('listening', cb));
     const address = this.server.address();
@@ -77,6 +81,7 @@ export class HttpServer {
   }
 
   private _onRequest(request: http.IncomingMessage, response: http.ServerResponse) {
+    this._log('web request', request.url);
     request.on('error', () => response.end());
     try {
       if (!request.url) {
@@ -84,6 +89,7 @@ export class HttpServer {
         return;
       }
       const url = new URL('http://localhost' + request.url);
+      this._log('url pathname', url.pathname);
       for (const route of this._routes) {
         if (route.exact && url.pathname === route.exact && route.handler(request, response))
           return;

packages/playwright-grid/src/grid/grid.ts

@@ -243,6 +243,8 @@ export class Grid {
       ws.on('error', e => this._log(e));
     });
     this._server.server.on('upgrade', async (request, socket, head) => {
+      this._log('upgrade', request.url, request.headers);
+
       if (this._accessKey && request.headers['x-playwright-access-key'] !== this._accessKey) {
         socket.destroy();
         return;
@@ -250,7 +252,6 @@ export class Grid {
 
       const url = new URL('http://internal' + request.url);
       const params = url.searchParams;
-      this._log(url.pathname);
 
       if (url.pathname.startsWith('/registerNode')) {
         const nodeRequest = new WebSocketRequest(this._wsServer, request, socket, head);

packages/playwright-grid/src/node/node.ts

@@ -28,19 +28,56 @@ const caps: Capabilities = {
 export class Node {
   workerSeq = 0;
 
-  constructor(grid: string, capacity: number, accessKey: string) {
+  constructor(readonly grid: string, readonly capacity: number, readonly accessKey: string) {
-    log('node created');
+    log('node created', accessKey);
-    const ws = new WebSocket(grid + `/registerNode?capacity=${capacity}&caps=${JSON.stringify(caps)}`, {
+  }
+
+  async connect() {
+    const wsGrid = 'ws://' + this.grid;
+    const url = wsGrid + `/registerNode?capacity=${this.capacity}&caps=${JSON.stringify(caps)}`;
+
+    for (let i = 0; i < 5; ++i) {
+      const ws = await this._connect(url);
+      if (ws) {
+        this._wire(ws, wsGrid);
+        return;
+      }
+      await new Promise(f => setTimeout(f, 5000));
+    }
+
+    // eslint-disable-next-line no-restricted-properties
+    process.exit(0);
+  }
+
+  private async _connect(url: string): Promise<WebSocket | null> {
+    return await new Promise(resolve => {
+      log('connecting', url);
+      const ws = new WebSocket(url, {
         headers: {
-        'x-playwright-access-key': accessKey,
+          'x-playwright-access-key': this.accessKey,
         }
       });
-    let nodeId = '';
       ws.on('error', error => {
         log(error);
+        resolve(null);
+      });
+      ws.on('open', () => {
+        log('connected', this.grid);
+        resolve(ws);
+      });
+    });
+  }
+
+  private _wire(ws: WebSocket, wsGrid: string) {
+    ws.on('close', () => {
       // eslint-disable-next-line no-restricted-properties
       process.exit(0);
     });
+    ws.on('error', () => {
+      // eslint-disable-next-line no-restricted-properties
+      process.exit(0);
+    });
+    let nodeId = '';
     ws.on('message', data => {
       const text = data.toString();
       const message = JSON.parse(text);
@@ -56,13 +93,11 @@ export class Node {
           ...process.env,
           PLAYWRIGHT_GRID_NODE_ID: nodeId,
           PLAYWRIGHT_GRID_WORKER_ID: workerId,
-          PLAYWRIGHT_GRID_ENDPOINT: grid,
+          PLAYWRIGHT_GRID_ENDPOINT: wsGrid,
-          PLAYWRIGHT_GRID_ACCESS_KEY: accessKey,
+          PLAYWRIGHT_GRID_ACCESS_KEY: this.accessKey,
         },
         detached: true
       });
     });
-    // eslint-disable-next-line no-restricted-properties
-    ws.on('close', () => process.exit(0));
   }
 }

packages/playwright-grid/src/node/worker.ts

@@ -16,11 +16,12 @@
 
 import debug from 'debug';
 import WebSocket from 'ws';
-import { DispatcherConnection, RootDispatcher, PlaywrightDispatcher, createPlaywright, serverSideCallMetadata } from 'playwright-core/lib/server';
+import { DispatcherConnection, RootDispatcher, PlaywrightDispatcher, createPlaywright, serverSideCallMetadata, SocksProxy } from 'playwright-core/lib/server';
-import { gracefullyProcessExitDoNotHang } from 'playwright-core/lib/utils';
+import { gracefullyCloseAll } from 'playwright-core/lib/utils';
+import type { Playwright } from 'playwright-core/lib/server';
 
 const workerId = process.env.PLAYWRIGHT_GRID_WORKER_ID!;
-const log = debug('pw:grid:browser@' + workerId);
+const log = debug('pw:grid:worker@' + workerId);
 
 class Worker {
   constructor() {
@@ -28,6 +29,20 @@ class Worker {
     const dispatcherConnection = new DispatcherConnection();
     let browserName: 'chromium' | 'webkit' | 'firefox';
     let launchOptions: any;
+    let proxyPattern: string | undefined;
+    let socksProxy: SocksProxy | undefined;
+
+    const dispose = async () => {
+      dispatcherConnection.onmessage = () => {};
+      // eslint-disable-next-line no-restricted-properties
+      setTimeout(() => process.exit(0), 30000);
+      await Promise.all([
+        socksProxy?.close(),
+        gracefullyCloseAll(),
+      ]).catch(() => {});
+      // eslint-disable-next-line no-restricted-properties
+      process.exit(0);
+    };
 
     const ws = new WebSocket(process.env.PLAYWRIGHT_GRID_ENDPOINT + `/registerWorker?nodeId=${process.env.PLAYWRIGHT_GRID_NODE_ID}&workerId=${workerId}`, {
       headers: {
@@ -42,29 +57,40 @@ class Worker {
 
       browserName = headers['x-playwright-browser'] as any || 'chromium';
       launchOptions = JSON.parse(headers['x-playwright-launch-options'] || '{}');
-      log('browserName', browserName);
+      proxyPattern = headers['x-playwright-proxy'] || '';
-      log('launchOptions', launchOptions);
+
+      log({ browserName, launchOptions, proxyPattern });
     });
     ws.once('open', () => {
       log('worker opened');
       new RootDispatcher(dispatcherConnection, async (rootScope, { sdkLanguage }) => {
         const playwright = createPlaywright({ sdkLanguage });
+        if (proxyPattern)
+          socksProxy = await createOwnedSocksProxy(proxyPattern, playwright);
         const browser = await playwright[browserName].launch(serverSideCallMetadata(), launchOptions);
-        return new PlaywrightDispatcher(rootScope, playwright, undefined, browser);
+        return new PlaywrightDispatcher(rootScope, playwright, socksProxy, browser);
       });
     });
     ws.on('message', message => dispatcherConnection.dispatch(JSON.parse(message.toString())));
     ws.on('error', error => {
       log('socket error');
-      dispatcherConnection.onmessage = () => {};
+      dispose();
-      gracefullyProcessExitDoNotHang(0);
     });
     ws.on('close', async () => {
       log('worker deleted');
-      dispatcherConnection.onmessage = () => {};
+      dispose();
-      gracefullyProcessExitDoNotHang(0);
     });
   }
 }
 
+async function createOwnedSocksProxy(proxyPattern: string, playwright: Playwright): Promise<SocksProxy | undefined> {
+  if (!proxyPattern)
+    return;
+  const socksProxy = new SocksProxy();
+  socksProxy.setPattern(proxyPattern);
+  playwright.options.socksProxyPort = await socksProxy.listen(0);
+  log(`started socks proxy on port ${playwright.options.socksProxyPort}`);
+  return socksProxy;
+}
+
 new Worker();

tests/library/playwright.config.ts

@@ -15,7 +15,7 @@
  */
 
 import { config as loadEnv } from 'dotenv';
-loadEnv({ path: path.join(__dirname, '..', '..', '.env') });
+loadEnv({ path: path.join(__dirname, '..', '..', '.env'), override: true });
 
 import type { Config, PlaywrightTestOptions, PlaywrightWorkerOptions, ReporterDescription } from '@playwright/test';
 import * as path from 'path';
@@ -78,23 +78,24 @@ if (mode === 'service2') {
 
 if (mode === 'service-grid') {
   connectOptions = {
-    wsEndpoint: 'ws://localhost:3333',
+    wsEndpoint: process.env.PLAYWRIGHT_GRID_URL || 'ws://localhost:3333',
     timeout: 60 * 60 * 1000,
     headers: {
-      'x-playwright-access-key': 'secret'
+      'x-playwright-access-key': process.env.PLAYWRIGHT_GRID_ACCESS_KEY || 'secret'
-    }
+    },
+    exposeNetwork: '<loopback>',
   };
-  webServer = [
+  webServer = process.env.PLAYWRIGHT_GRID_URL ? [] : [
     {
       command: 'node ../../packages/playwright-grid/cli.js grid --port=3333 --access-key=secret',
       stdout: 'pipe',
       url: 'http://localhost:3333/secret',
       reuseExistingServer: !process.env.CI,
     }, {
-      command: 'node ../../packages/playwright-grid/cli.js node --grid=ws://localhost:3333 --access-key=secret --capacity=2',
+      command: 'node ../../packages/playwright-grid/cli.js node --grid=localhost:3333 --access-key=secret --capacity=2',
     },
     {
-      command: 'node ../../packages/playwright-grid/cli.js node --grid=ws://localhost:3333 --access-key=secret --capacity=2',
+      command: 'node ../../packages/playwright-grid/cli.js node --grid=localhost:3333 --access-key=secret --capacity=2',
     }
   ];
 }

utils/workspace.js

@@ -33,6 +33,7 @@ class PWPackage {
     this.name = descriptor.name;
     this.path = descriptor.path;
     this.files = descriptor.files;
+    this.noConsistent = descriptor.noConsistent;
     this.packageJSONPath = path.join(this.path, 'package.json');
     this.packageJSON = JSON.parse(fs.readFileSync(this.packageJSONPath, 'utf8'));
     this.isPrivate = !!this.packageJSON.private;
@@ -120,6 +121,10 @@ class Workspace {
         pkg.packageJSON.author = workspacePackageJSON.author;
         pkg.packageJSON.license = workspacePackageJSON.license;
       }
+
+      if (pkg.noConsistent)
+        continue;
+
       for (const otherPackage of this._packages) {
         if (pkgLockEntry.dependencies && pkgLockEntry.dependencies[otherPackage.name])
           pkgLockEntry.dependencies[otherPackage.name] = version;
@@ -212,6 +217,7 @@ const workspace = new Workspace(ROOT_PATH, [
     name: '@playwright/experimental-grid',
     path: path.join(ROOT_PATH, 'packages', 'playwright-grid'),
     files: ['LICENSE'],
+    noConsistent: true,
   }),
 ]);