diff --git a/.github/workflows/tests_secondary.yml b/.github/workflows/tests_secondary.yml index d6dd58fc12..426c0a754c 100644 --- a/.github/workflows/tests_secondary.yml +++ b/.github/workflows/tests_secondary.yml @@ -802,34 +802,3 @@ jobs: - run: npx playwright install-deps - run: utils/build/build-playwright-driver.sh - smoke_test_docker_integration: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: 14 - - run: npm i -g npm@8 - - run: npm ci - env: - DEBUG: pw:install - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - - run: npm run build - - run: npx playwright install --with-deps - - run: | - ./utils/docker/build.sh --amd64 focal $PWTEST_DOCKER_BASE_IMAGE - npx playwright docker build - nohup npx playwright docker start & - xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --grep '@smoke' - env: - PLAYWRIGHT_DOCKER: 1 - PWTEST_DOCKER_BASE_IMAGE: playwright:localbuild - - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json - if: always() - shell: bash - - uses: actions/upload-artifact@v3 - if: always() - with: - name: docker-integration-test-results - path: test-results - diff --git a/packages/playwright-core/bin/container_install_deps.sh b/packages/playwright-core/bin/container_install_deps.sh deleted file mode 100755 index b0f10c5888..0000000000 --- a/packages/playwright-core/bin/container_install_deps.sh +++ /dev/null @@ -1,85 +0,0 @@ -export NOVNC_REF='1.3.0' -export WEBSOCKIFY_REF='0.10.0' -export DEBIAN_FRONTEND=noninteractive - -# Install FluxBox, VNC & noVNC -mkdir -p /opt/bin && chmod +x /dev/shm \ - && apt-get update && apt-get install -y unzip fluxbox x11vnc \ - && curl -L -o noVNC.zip "https://github.com/novnc/noVNC/archive/v${NOVNC_REF}.zip" \ - && unzip -x noVNC.zip \ - && rm -rf noVNC-${NOVNC_REF}/{docs,tests} \ - && mv noVNC-${NOVNC_REF} /opt/bin/noVNC \ - && cp /opt/bin/noVNC/vnc.html /opt/bin/noVNC/index.html \ - && rm noVNC.zip \ - && curl -L -o websockify.zip "https://github.com/novnc/websockify/archive/v${WEBSOCKIFY_REF}.zip" \ - && unzip -x websockify.zip \ - && rm websockify.zip \ - && rm -rf websockify-${WEBSOCKIFY_REF}/{docs,tests} \ - && mv websockify-${WEBSOCKIFY_REF} /opt/bin/noVNC/utils/websockify - -# Patch noVNC - -cat <<'EOF' > /opt/bin/noVNC/clip.patch -diff --git a/app/ui.js b/app/ui.js -index cb6a9fd..dbe42e0 100644 ---- a/app/ui.js -+++ b/app/ui.js -@@ -951,6 +951,7 @@ const UI = { - clipboardReceive(e) { - Log.Debug(">> UI.clipboardReceive: " + e.detail.text.substr(0, 40) + "..."); - document.getElementById('noVNC_clipboard_text').value = e.detail.text; -+ navigator.clipboard.writeText(e.detail.text).catch(() => {}); - Log.Debug("<< UI.clipboardReceive"); - }, - -diff --git a/core/rfb.js b/core/rfb.js -index ea3bf58..fad57bc 100644 ---- a/core/rfb.js -+++ b/core/rfb.js -@@ -176,6 +176,7 @@ export default class RFB extends EventTargetMixin { - handleMouse: this._handleMouse.bind(this), - handleWheel: this._handleWheel.bind(this), - handleGesture: this._handleGesture.bind(this), -+ handleFocus: () => navigator.clipboard.readText().then(this.clipboardPasteFrom.bind(this)).catch(() => {}) - }; - - // main setup -@@ -515,6 +516,7 @@ export default class RFB extends EventTargetMixin { - this._canvas.addEventListener("gesturestart", this._eventHandlers.handleGesture); - this._canvas.addEventListener("gesturemove", this._eventHandlers.handleGesture); - this._canvas.addEventListener("gestureend", this._eventHandlers.handleGesture); -+ window.addEventListener('focus', this._eventHandlers.handleFocus); - - Log.Debug("<< RFB.connect"); - } -@@ -522,6 +524,7 @@ export default class RFB extends EventTargetMixin { - _disconnect() { - Log.Debug(">> RFB.disconnect"); - this._cursor.detach(); -+ window.removeEventListener('focus', this._eventHandlers.handleFocus); - this._canvas.removeEventListener("gesturestart", this._eventHandlers.handleGesture); - this._canvas.removeEventListener("gesturemove", this._eventHandlers.handleGesture); - this._canvas.removeEventListener("gestureend", this._eventHandlers.handleGesture); -EOF - -cd /opt/bin/noVNC -git apply clip.patch - -# Configure FluxBox menus -mkdir /root/.fluxbox -cat <<'EOF' > /root/.fluxbox/menu - [begin] (fluxbox) - [submenu] (Browsers) {} - [exec] (Chromium) { /ms-playwright-agent/node_modules/.bin/playwright docker launch --endpoint http://127.0.0.1:5400 --browser chromium } <> - [exec] (Firefox) { /ms-playwright-agent/node_modules/.bin/playwright docker launch --endpoint http://127.0.0.1:5400 --browser firefox } <> - [exec] (WebKit) { /ms-playwright-agent/node_modules/.bin/playwright docker launch --endpoint http://127.0.0.1:5400 --browser webkit } <> - [end] - [include] (/etc/X11/fluxbox/fluxbox-menu) - [end] -EOF - -cat <<'EOF' > /root/.fluxbox/lastwallpaper -$center $full|/ms-playwright-agent/node_modules/playwright-core/lib/server/chromium/appIcon.png||:99 -$center $full|/ms-playwright-agent/node_modules/playwright-core/lib/server/chromium/appIcon.png||:99.0 -EOF - diff --git a/packages/playwright-core/bin/container_landing.html b/packages/playwright-core/bin/container_landing.html deleted file mode 100644 index e1e41b0e18..0000000000 --- a/packages/playwright-core/bin/container_landing.html +++ /dev/null @@ -1,35 +0,0 @@ - - - -

Playwright Container

- View Screen - diff --git a/packages/playwright-core/bin/container_novnc_proxy.js b/packages/playwright-core/bin/container_novnc_proxy.js deleted file mode 100644 index ab1fa1504d..0000000000 --- a/packages/playwright-core/bin/container_novnc_proxy.js +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const http = require('http'); -const fs = require('fs'); -const path = require('path'); -const { debug, program } = require('../lib/utilsBundle'); -const { ProxyServer } = require('../lib/third_party/http_proxy'); - -const debugLog = debug('pw:proxy'); - -program - .command('start') - .description('reverse proxy for novnc and playwright server') - .option('--port ', 'port number') - .option('--server-endpoint ', 'Playwright Server endpoint') - .option('--novnc-endpoint ', 'novnc server endpoint') - .option('--novnc-ws-path ', 'novnc websocket path') - .action(async function(options) { - launchReverseProxy(options.port, options.serverEndpoint, options.novncEndpoint, options.novncWsPath); - }); - -program.parse(process.argv); - -async function launchReverseProxy(port, serverEndpoint, novncEndpoint, novncWSPath) { - const vncProxy = new ProxyServer(novncEndpoint, debugLog); - const serverProxy = new ProxyServer(serverEndpoint, debugLog); - - const httpServer = http.createServer((request, response) => { - if (request.url === '/' && request.method === 'GET') { - response.writeHead(200, { - 'content-type': 'text/html', - }).end(fs.readFileSync(path.join(__dirname, 'container_landing.html'), 'utf-8')); - } else if ((request.url === '/screen' || request.url === '/screen/') && request.method === 'GET') { - response.writeHead(307, { - Location: `/screen/?resize=scale&autoconnect=1&path=${novncWSPath}`, - }).end(); - } else if (request.url?.startsWith('/screen')) { - request.url = request.url.substring('/screen'.length); - vncProxy.web(request, response); - } else { - serverProxy.web(request, response); - } - }); - httpServer.on('error', error => debugLog(error)); - httpServer.on('upgrade', (request, socket, head) => { - if (request.url === '/' + novncWSPath) - vncProxy.ws(request, socket, head); - else - serverProxy.ws(request, socket, head); - }); - httpServer.listen(port, () => { - console.log('Playwright container listening on', port); - }); -} - diff --git a/packages/playwright-core/bin/container_run_server.sh b/packages/playwright-core/bin/container_run_server.sh deleted file mode 100755 index bfdf3666ab..0000000000 --- a/packages/playwright-core/bin/container_run_server.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -set -e - -trap "cd $(pwd -P)" EXIT -cd "$(dirname "$0")" - -SCREEN_WIDTH=1360 -SCREEN_HEIGHT=1020 -SCREEN_DEPTH=24 -SCREEN_DPI=96 -GEOMETRY="$SCREEN_WIDTH""x""$SCREEN_HEIGHT""x""$SCREEN_DEPTH" - -nohup /usr/bin/xvfb-run --server-num=$DISPLAY_NUM \ - --listen-tcp \ - --server-args="-screen 0 "$GEOMETRY" -fbdir /var/tmp -dpi "$SCREEN_DPI" -listen tcp -noreset -ac +extension RANDR" \ - /usr/bin/fluxbox -display "$DISPLAY" >/dev/null 2>&1 & - -for i in $(seq 1 500); do - if xdpyinfo -display $DISPLAY >/dev/null 2>&1; then - break - fi - echo "Waiting for Xvfb..." - sleep 0.2 -done - -# Launch x11 -nohup x11vnc -noprimary -nosetprimary -forever -shared -rfbport 5900 -rfbportv6 5900 -display "$DISPLAY" >/dev/null 2>&1 & -# Launch novnc -nohup /opt/bin/noVNC/utils/novnc_proxy --listen 7900 --vnc localhost:5900 >/dev/null 2>&1 & -# Launch reverse proxy -NOVNC_UUID=$(cat /proc/sys/kernel/random/uuid) -node ./container_novnc_proxy.js start --server-endpoint="http://127.0.0.1:5200" --novnc-endpoint="http://127.0.0.1:7900" --novnc-ws-path="${NOVNC_UUID}" --port 5400 & - -cd /ms-playwright-agent - -PW_UUID=$(cat /proc/sys/kernel/random/uuid) - -# Make sure to re-start playwright server if something goes wrong. -# The approach taken from: https://stackoverflow.com/a/697064/314883 - -until npx playwright run-server --port=5200 --path=/$PW_UUID; do - echo "Server crashed with exit code $?. Respawning.." >&2 - sleep 1 -done - diff --git a/packages/playwright-core/src/cli/cli.ts b/packages/playwright-core/src/cli/cli.ts index 9f424c6bae..8b92174eb3 100755 --- a/packages/playwright-core/src/cli/cli.ts +++ b/packages/playwright-core/src/cli/cli.ts @@ -38,7 +38,6 @@ import type { GridFactory } from '../grid/gridServer'; import { GridServer } from '../grid/gridServer'; import type { Executable } from '../server'; import { registry, writeDockerVersion } from '../server'; -import { addDockerCLI } from '../containers/docker'; const packageJSON = require('../../package.json'); @@ -312,8 +311,6 @@ Examples: $ show-trace https://example.com/trace.zip`); -addDockerCLI(program); - if (!process.env.PW_LANG_NAME) { let playwrightTestPackagePath = null; const resolvePwTestPaths = [__dirname, process.cwd()]; diff --git a/packages/playwright-core/src/containers/DEPS.list b/packages/playwright-core/src/containers/DEPS.list deleted file mode 100644 index 6d98651fb8..0000000000 --- a/packages/playwright-core/src/containers/DEPS.list +++ /dev/null @@ -1,7 +0,0 @@ -[*] -../utils/ -../utilsBundle.ts -../common/ -../server/ -../server/dispatchers/ -../.. diff --git a/packages/playwright-core/src/containers/docker.ts b/packages/playwright-core/src/containers/docker.ts deleted file mode 100644 index 1e0d9a4c24..0000000000 --- a/packages/playwright-core/src/containers/docker.ts +++ /dev/null @@ -1,429 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the 'License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* eslint-disable no-console */ - -import path from 'path'; -import { spawnAsync } from '../utils/spawnAsync'; -import * as utils from '../utils'; -import { getPlaywrightVersion, getUserAgent } from '../utils/userAgent'; -import { urlToWSEndpoint } from '../server/dispatchers/localUtilsDispatcher'; -import { WebSocketTransport } from '../server/transport'; -import { SocksInterceptor } from '../server/socksInterceptor'; -import * as dockerApi from './dockerApi'; -import type { Command } from '../utilsBundle'; -import * as playwright from '../..'; - -const VRT_IMAGE_DISTRO = 'focal'; -const VRT_IMAGE_NAME = `playwright:local-${getPlaywrightVersion()}-${VRT_IMAGE_DISTRO}`; -const VRT_CONTAINER_NAME = `playwright-${getPlaywrightVersion()}-${VRT_IMAGE_DISTRO}`; -const VRT_CONTAINER_LABEL_NAME = 'dev.playwright.vrt-service.version'; -const VRT_CONTAINER_LABEL_VALUE = '1'; - -async function startPlaywrightContainer(port: number) { - await checkDockerEngineIsRunningOrDie(); - - await stopAllPlaywrightContainers(); - - process.stdout.write(`Starting docker container... `); - const time = Date.now(); - const info = await ensurePlaywrightContainerOrDie(port); - const deltaMs = (Date.now() - time); - console.log('Done in ' + (deltaMs / 1000).toFixed(1) + 's'); - await tetherHostNetwork(info.httpEndpoint); - console.log('Endpoint:', info.httpEndpoint); -} - -async function stopAllPlaywrightContainers() { - await checkDockerEngineIsRunningOrDie(); - - const allContainers = await dockerApi.listContainers(); - const vrtContainers = allContainers.filter(container => container.labels[VRT_CONTAINER_LABEL_NAME] === VRT_CONTAINER_LABEL_VALUE); - await Promise.all(vrtContainers.map(container => dockerApi.stopContainer({ - containerId: container.containerId, - waitUntil: 'removed', - }))); -} - -async function deletePlaywrightImage() { - await checkDockerEngineIsRunningOrDie(); - - const dockerImage = await findDockerImage(VRT_IMAGE_NAME); - if (!dockerImage) - return; - - if (await containerInfo()) - await stopAllPlaywrightContainers(); - await dockerApi.removeImage(dockerImage.imageId); -} - -async function buildPlaywrightImage() { - await checkDockerEngineIsRunningOrDie(); - - // 1. Build or pull base image. - let baseImageName = process.env.PWTEST_DOCKER_BASE_IMAGE || ''; - if (!baseImageName) { - const isDevelopmentMode = getPlaywrightVersion().includes('next'); - if (isDevelopmentMode) { - // Use our docker build scripts in development mode! - const arch = process.arch === 'arm64' ? '--arm64' : '--amd64'; - throw createStacklessError(utils.wrapInASCIIBox([ - `You are in DEVELOPMENT mode!`, - ``, - `1. Build local base image`, - ` ./utils/docker/build.sh ${arch} ${VRT_IMAGE_DISTRO} playwright:localbuild`, - `2. Use the local base to build VRT image:`, - ` PWTEST_DOCKER_BASE_IMAGE=playwright:localbuild npx playwright docker build`, - ].join('\n'), 1)); - } - baseImageName = `mcr.microsoft.com/playwright:v${getPlaywrightVersion()}-${VRT_IMAGE_DISTRO}`; - const { code } = await spawnAsync('docker', ['pull', baseImageName], { stdio: 'inherit' }); - if (code !== 0) - throw new Error('Failed to pull docker image!'); - } - // 2. Find pulled docker image - const dockerImage = await findDockerImage(baseImageName); - if (!dockerImage) - throw new Error(`Failed to pull ${baseImageName}`); - // 3. Delete previous build of the playwright image to avoid untagged images. - await deletePlaywrightImage(); - // 4. Launch container and install VNC in it - console.log(`Building ${VRT_IMAGE_NAME}...`); - const containerId = await dockerApi.launchContainer({ - imageId: dockerImage.imageId, - autoRemove: false, - workingDir: '/ms-playwright-agent', - command: ['npx', 'playwright', 'docker', 'install-server-deps'], - waitUntil: 'not-running', - }); - - // 4. Commit a new image based on the launched container with installed VNC & noVNC. - const [vrtRepo, vrtTag] = VRT_IMAGE_NAME.split(':'); - await dockerApi.commitContainer({ - containerId, - repo: vrtRepo, - tag: vrtTag, - workingDir: '/ms-playwright-agent', - entrypoint: ['npx', 'playwright', 'docker', 'run-server'], - env: { - 'DISPLAY_NUM': '99', - 'DISPLAY': ':99', - }, - }); - await dockerApi.removeContainer(containerId); - console.log(`Done!`); -} - -interface ContainerInfo { - httpEndpoint: string; -} - -async function printDockerStatus() { - const isDockerEngine = await dockerApi.checkEngineRunning(); - const imageIsPulled = isDockerEngine && !!(await findDockerImage(VRT_IMAGE_NAME)); - const info = isDockerEngine ? await containerInfo() : undefined; - console.log(JSON.stringify({ - dockerEngineRunning: isDockerEngine, - imageName: VRT_IMAGE_NAME, - imageIsPulled, - containerEndpoint: info?.httpEndpoint ?? '', - }, null, 2)); -} - -export async function containerInfo(): Promise { - const allContainers = await dockerApi.listContainers(); - const pwDockerImage = await findDockerImage(VRT_IMAGE_NAME); - const container = allContainers.find(container => container.imageId === pwDockerImage?.imageId && container.state === 'running'); - if (!container) - return undefined; - const logLines = await dockerApi.getContainerLogs(container.containerId); - - const containerUrlToHostUrl = (address: string) => { - const url = new URL(address); - const portBinding = container.portBindings.find(binding => binding.containerPort === +url.port); - if (!portBinding) - return undefined; - - url.host = portBinding.ip; - url.port = portBinding.hostPort + ''; - return url.toString(); - }; - - const WS_LINE_PREFIX = 'Listening on ws://'; - const REVERSE_PROXY_LINE_PREFIX = 'Playwright container listening on'; - const webSocketLine = logLines.find(line => line.startsWith(WS_LINE_PREFIX)); - const reverseProxyLine = logLines.find(line => line.startsWith(REVERSE_PROXY_LINE_PREFIX)); - if (!webSocketLine || !reverseProxyLine) - return undefined; - const httpEndpoint = containerUrlToHostUrl('http://127.0.0.1:' + reverseProxyLine.substring(REVERSE_PROXY_LINE_PREFIX.length).trim()); - return httpEndpoint ? { httpEndpoint } : undefined; -} - -export async function ensurePlaywrightContainerOrDie(port: number): Promise { - const pwImage = await findDockerImage(VRT_IMAGE_NAME); - if (!pwImage) { - throw createStacklessError('\n' + utils.wrapInASCIIBox([ - `Failed to find local docker image.`, - `Please build local docker image with the following command:`, - ``, - ` npx playwright docker build`, - ``, - `<3 Playwright Team`, - ].join('\n'), 1)); - } - - let info = await containerInfo(); - if (info) - return info; - - // The `npx playwright docker build` command is *NOT GUARANTEED* to produce - // images with the same SHA. - // - // Consider the following sequence of actions: - // 1. Build first version of image: `npx playwright docker build` - // 2. Run container off the image: `npx playwright docker start` - // 3. Build second version of image: `npx playwright docker build` - // - // Our container auto-detection is based on the parent image SHA. - // If the image produced at Step 3 has a different SHA then the one produced on Step 1, - // then we **won't be able** to auto-detect the container from Step 2. - // - // Additionally, we won't be able to launch a new container based off image - // from Step 3, since it will have a conflicting container name. - // - // We check if there's a same-named container running to detect & handle this situation. - const hasSameNamedContainer = async () => (await dockerApi.listContainers()).some(container => container.names.includes(VRT_CONTAINER_NAME)); - if (await hasSameNamedContainer()) { - // Since we mark all our containers with labels, we'll be able to stop it. - await stopAllPlaywrightContainers(); - // If it wasn't our container, then it was launched manually and has to be - // stopped manually as well. - if (await hasSameNamedContainer()) { - throw createStacklessError('\n' + utils.wrapInASCIIBox([ - `There is already a container with name ${VRT_CONTAINER_NAME}`, - `Please stop this container manually and rerun tests:`, - ``, - ` docker kill ${VRT_CONTAINER_NAME}`, - ``, - `<3 Playwright Team`, - ].join('\n'), 1)); - } - } - - const env: Record = { - DEBUG: process.env.DEBUG, - }; - for (const [key, value] of Object.entries(process.env)) { - if (key.startsWith('PLAYWRIGHT_')) - env[key] = value; - } - await dockerApi.launchContainer({ - imageId: pwImage.imageId, - name: VRT_CONTAINER_NAME, - autoRemove: true, - ports: [ - { container: 5400, host: port }, - ], - labels: { - [VRT_CONTAINER_LABEL_NAME]: VRT_CONTAINER_LABEL_VALUE, - }, - env, - }); - - // Wait for the service to become available. - const startTime = Date.now(); - const timeouts = [0, 100, 100, 200, 500, 1000]; - do { - await new Promise(x => setTimeout(x, timeouts.shift() ?? 1000)); - info = await containerInfo(); - } while (!info && Date.now() < startTime + 60000); - - if (!info) - throw new Error('Failed to launch docker container!'); - return info; -} - -export async function checkDockerEngineIsRunningOrDie() { - if (await dockerApi.checkEngineRunning()) - return; - throw createStacklessError(utils.wrapInASCIIBox([ - `Docker is not running!`, - `Please install and launch docker:`, - ``, - ` https://docs.docker.com/get-docker`, - ``, - ].join('\n'), 1)); -} - -async function findDockerImage(imageName: string): Promise { - const images = await dockerApi.listImages(); - return images.find(image => image.names.includes(imageName)); -} - -function createStacklessError(message: string) { - const error = new Error(message); - error.stack = ''; - return error; -} - -async function tetherHostNetwork(endpoint: string) { - const wsEndpoint = await urlToWSEndpoint(undefined /* progress */, endpoint); - - const headers: any = { - 'User-Agent': getUserAgent(), - 'x-playwright-network-tethering': '1', - 'x-playwright-proxy': '*', - }; - const transport = await WebSocketTransport.connect(undefined /* progress */, wsEndpoint, headers, true /* followRedirects */); - const socksInterceptor = new SocksInterceptor(transport, '*', undefined); - transport.onmessage = json => socksInterceptor.interceptMessage(json); - transport.onclose = () => { - socksInterceptor.cleanup(); - }; - await transport.send({ - id: 1, - guid: '', - method: 'initialize', - params: { - 'sdkLanguage': 'javascript' - }, - metadata: { - stack: [], - apiName: '', - internal: true - }, - } as any); -} - -export function addDockerCLI(program: Command) { - const dockerCommand = program.command('docker', { hidden: true }) - .description(`Manage Docker integration (EXPERIMENTAL)`); - - dockerCommand.command('build') - .description('build local docker image') - .action(async function(options) { - try { - await buildPlaywrightImage(); - } catch (e) { - console.error(e.stack ? e : e.message); - process.exit(1); - } - }); - - dockerCommand.command('start') - .description('start docker container') - .option('--port ', 'port to start container on. Auto-pick by default') - .action(async function(options) { - try { - const port = options.port ? +options.port : 0; - if (isNaN(port)) { - console.error(`ERROR: bad port number "${options.port}"`); - process.exit(1); - } - await startPlaywrightContainer(port); - } catch (e) { - console.error(e.stack ? e : e.message); - process.exit(1); - } - }); - - dockerCommand.command('stop') - .description('stop docker container') - .action(async function(options) { - try { - await stopAllPlaywrightContainers(); - } catch (e) { - console.error(e.stack ? e : e.message); - process.exit(1); - } - }); - - dockerCommand.command('delete-image', { hidden: true }) - .description('delete docker image, if any') - .action(async function(options) { - try { - await deletePlaywrightImage(); - } catch (e) { - console.error(e.stack ? e : e.message); - process.exit(1); - } - }); - - dockerCommand.command('install-server-deps', { hidden: true }) - .description('install run-server dependencies') - .action(async function() { - const { code } = await spawnAsync('bash', [path.join(__dirname, '..', '..', 'bin', 'container_install_deps.sh')], { stdio: 'inherit' }); - if (code !== 0) - throw new Error('Failed to install server dependencies!'); - }); - - dockerCommand.command('run-server', { hidden: true }) - .description('run playwright server') - .action(async function() { - await spawnAsync('bash', [path.join(__dirname, '..', '..', 'bin', 'container_run_server.sh')], { stdio: 'inherit' }); - }); - - dockerCommand.command('print-status-json', { hidden: true }) - .description('print docker status') - .action(async function(options) { - await printDockerStatus(); - }); - - dockerCommand.command('launch', { hidden: true }) - .description('launch browser in container') - .option('--browser ', 'browser to launch') - .option('--endpoint ', 'server endpoint') - .action(async function(options: { browser: string, endpoint: string }) { - let browserType: playwright.BrowserType | undefined; - if (options.browser === 'chromium') - browserType = playwright.chromium; - else if (options.browser === 'firefox') - browserType = playwright.firefox; - else if (options.browser === 'webkit') - browserType = playwright.webkit; - if (!browserType) { - console.error('Unknown browser name: ', options.browser); - process.exit(1); - } - const browser = await browserType.connect(options.endpoint, { - headers: { - 'x-playwright-launch-options': JSON.stringify({ - headless: false, - viewport: null, - }), - }, - _exposeNetwork: '*', - } as any); - const context = await browser.newContext(); - context.on('page', (page: playwright.Page) => { - page.on('dialog', () => {}); // Prevent dialogs from being automatically dismissed. - page.once('close', () => { - const hasPage = browser.contexts().some((context: playwright.BrowserContext) => context.pages().length > 0); - if (hasPage) - return; - // Avoid the error when the last page is closed because the browser has been closed. - browser.close().catch((e: Error) => null); - }); - }); - await context.newPage(); - }); - - dockerCommand.command('tether', { hidden: true }) - .description('tether local network to the playwright server') - .option('--endpoint ', 'server endpoint') - .action(async function(options: { endpoint: string }) { - await tetherHostNetwork(options.endpoint); - }); -} diff --git a/packages/playwright-core/src/containers/dockerApi.ts b/packages/playwright-core/src/containers/dockerApi.ts deleted file mode 100644 index 95cb22f3d0..0000000000 --- a/packages/playwright-core/src/containers/dockerApi.ts +++ /dev/null @@ -1,228 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the 'License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import http from 'http'; - -// Docker engine API. -// See https://docs.docker.com/engine/api/v1.41/ - -const DOCKER_API_VERSION = '1.41'; - -export interface DockerImage { - imageId: string; - names: string[]; -} - -export interface PortBinding { - ip: string; - hostPort: number; - containerPort: number; -} - -export interface DockerContainer { - containerId: string; - labels: Record; - imageId: string; - state: 'created'|'restarting'|'running'|'removing'|'paused'|'exited'|'dead'; - names: string[]; - portBindings: PortBinding[]; -} - -export async function listContainers(): Promise { - const containers = (await getJSON('/containers/json')) ?? []; - return containers.map((container: any) => ({ - containerId: container.Id, - imageId: container.ImageID, - state: container.State, - // Note: container names are usually prefixed with '/'. - // See https://github.com/moby/moby/issues/6705 - names: (container.Names ?? []).map((name: string) => name.startsWith('/') ? name.substring(1) : name), - portBindings: container.Ports?.map((portInfo: any) => ({ - ip: portInfo.IP, - hostPort: portInfo.PublicPort, - containerPort: portInfo.PrivatePort, - })) ?? [], - labels: container.Labels ?? {}, - })); -} - -interface LaunchContainerOptions { - imageId: string; - autoRemove: boolean; - command?: string[]; - labels?: Record; - ports?: { container: number, host: number }[], - name?: string; - workingDir?: string; - waitUntil?: 'not-running' | 'next-exit' | 'removed'; - env?: { [key: string]: string | number | boolean | undefined }; -} - -export async function launchContainer(options: LaunchContainerOptions): Promise { - const ExposedPorts: any = {}; - const PortBindings: any = {}; - for (const port of (options.ports ?? [])) { - ExposedPorts[`${port.container}/tcp`] = {}; - PortBindings[`${port.container}/tcp`] = [{ HostPort: port.host + '', HostIp: '127.0.0.1' }]; - } - const container = await postJSON(`/containers/create` + (options.name ? '?name=' + options.name : ''), { - Cmd: options.command, - WorkingDir: options.workingDir, - Labels: options.labels ?? {}, - AttachStdout: true, - AttachStderr: true, - Image: options.imageId, - ExposedPorts, - Env: dockerProtocolEnv(options.env), - HostConfig: { - Init: true, - AutoRemove: options.autoRemove, - ShmSize: 2 * 1024 * 1024 * 1024, - PortBindings, - }, - }); - await postJSON(`/containers/${container.Id}/start`); - if (options.waitUntil) - await postJSON(`/containers/${container.Id}/wait?condition=${options.waitUntil}`); - return container.Id; -} - -interface StopContainerOptions { - containerId: string, - waitUntil?: 'not-running' | 'next-exit' | 'removed'; -} - -export async function stopContainer(options: StopContainerOptions) { - await Promise.all([ - // Make sure to wait for the container to be removed. - postJSON(`/containers/${options.containerId}/wait?condition=${options.waitUntil ?? 'not-running'}`), - postJSON(`/containers/${options.containerId}/kill`), - ]); -} - -export async function removeContainer(containerId: string) { - await Promise.all([ - // Make sure to wait for the container to be removed. - postJSON(`/containers/${containerId}/wait?condition=removed`), - callDockerAPI('delete', `/containers/${containerId}`), - ]); -} - -export async function getContainerLogs(containerId: string): Promise { - const rawLogs = await callDockerAPI('get', `/containers/${containerId}/logs?stdout=true&stderr=true`).catch(e => ''); - if (!rawLogs) - return []; - // Docker might prefix every log line with 8 characters. Stip them out. - // See https://github.com/moby/moby/issues/7375 - // This doesn't happen if the containers is launched manually with attached terminal. - return rawLogs.split('\n').map(line => { - if ([0, 1, 2].includes(line.charCodeAt(0))) - return line.substring(8); - return line; - }); -} - -interface CommitContainerOptions { - containerId: string, - repo: string, - tag: string, - entrypoint?: string[], - workingDir?: string, - env?: {[key: string]: string | number | boolean | undefined}, -} - -function dockerProtocolEnv(env?: {[key: string]: string | number | boolean | undefined}): string[] { - const result = []; - for (const [key, value] of Object.entries(env ?? {})) - result.push(`${key}=${value}`); - return result; -} - -export async function commitContainer(options: CommitContainerOptions) { - await postJSON(`/commit?container=${options.containerId}&repo=${options.repo}&tag=${options.tag}`, { - Entrypoint: options.entrypoint, - WorkingDir: options.workingDir, - Env: dockerProtocolEnv(options.env), - }); -} - -export async function listImages(): Promise { - const rawImages: any[] = (await getJSON('/images/json')) ?? []; - return rawImages.map((rawImage: any) => ({ - imageId: rawImage.Id, - names: rawImage.RepoTags ?? [], - })); -} - -export async function removeImage(imageId: string) { - await callDockerAPI('delete', `/images/${imageId}`); -} - -export async function checkEngineRunning() { - try { - await callDockerAPI('get', '/info'); - return true; - } catch (e) { - return false; - } -} - -async function getJSON(url: string): Promise { - const result = await callDockerAPI('get', url); - if (!result) - return result; - return JSON.parse(result); -} - -async function postJSON(url: string, json: any = undefined) { - const result = await callDockerAPI('post', url, json ? JSON.stringify(json) : undefined); - if (!result) - return result; - return JSON.parse(result); -} - -function callDockerAPI(method: 'post'|'get'|'delete', url: string, body: Buffer|string|undefined = undefined): Promise { - const dockerSocket = process.platform === 'win32' ? '\\\\.\\pipe\\docker_engine' : '/var/run/docker.sock'; - return new Promise((resolve, reject) => { - const request = http.request({ - socketPath: dockerSocket, - path: `/v${DOCKER_API_VERSION}${url}`, - timeout: 30000, - method, - }, (response: http.IncomingMessage) => { - let body = ''; - response.on('data', function(chunk){ - body += chunk; - }); - response.on('end', function(){ - if (!response.statusCode || response.statusCode < 200 || response.statusCode >= 300) - reject(new Error(`${method} ${url} FAILED with statusCode ${response.statusCode} and body\n${body}`)); - else - resolve(body); - }); - }); - request.on('error', function(e){ - reject(e); - }); - if (body) { - request.setHeader('Content-Type', 'application/json'); - request.setHeader('Content-Length', body.length); - request.write(body); - } else { - request.setHeader('Content-Type', 'text/plain'); - } - request.end(); - }); -} diff --git a/packages/playwright-test/src/plugins/dockerPlugin.ts b/packages/playwright-test/src/plugins/dockerPlugin.ts deleted file mode 100644 index 399dfe8147..0000000000 --- a/packages/playwright-test/src/plugins/dockerPlugin.ts +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import type { TestRunnerPlugin } from '.'; -import type { FullConfig, Reporter } from '../../types/testReporter'; -import { colors } from 'playwright-core/lib/utilsBundle'; -import { checkDockerEngineIsRunningOrDie, containerInfo } from 'playwright-core/lib/containers/docker'; - -export const dockerPlugin: TestRunnerPlugin = { - name: 'playwright:docker', - - async setup(config: FullConfig, configDir: string, reporter: Reporter) { - if (!process.env.PLAYWRIGHT_DOCKER) - return; - - const println = (text: string) => reporter.onStdOut?.(text + '\n'); - - println(colors.dim('Using docker container to run browsers.')); - await checkDockerEngineIsRunningOrDie(); - const info = await containerInfo(); - if (!info) - throw new Error('ERROR: please launch docker container separately!'); - println(''); - process.env.PW_TEST_CONNECT_WS_ENDPOINT = info.httpEndpoint; - process.env.PW_TEST_CONNECT_EXPOSE_NETWORK = '*'; - }, -}; - - diff --git a/packages/playwright-test/src/runner/runner.ts b/packages/playwright-test/src/runner/runner.ts index 31664f9455..a25afaaa03 100644 --- a/packages/playwright-test/src/runner/runner.ts +++ b/packages/playwright-test/src/runner/runner.ts @@ -17,7 +17,6 @@ import { monotonicTime } from 'playwright-core/lib/utils'; import type { FullResult } from '../../types/testReporter'; -import { dockerPlugin } from '../plugins/dockerPlugin'; import { webServerPluginsForConfig } from '../plugins/webServerPlugin'; import { collectFilesForProject, filterProjects } from './projectUtils'; import { createReporter } from './reporters'; @@ -54,8 +53,6 @@ export class Runner { // Legacy webServer support. webServerPluginsForConfig(config).forEach(p => config._internal.plugins.push({ factory: p })); - // Docker support. - config._internal.plugins.push({ factory: dockerPlugin }); const reporter = await createReporter(config, listOnly); const taskRunner = listOnly ? createTaskRunnerForList(config, reporter) diff --git a/tests/config/platformFixtures.ts b/tests/config/platformFixtures.ts index bd318803ea..4a044c8dcc 100644 --- a/tests/config/platformFixtures.ts +++ b/tests/config/platformFixtures.ts @@ -24,7 +24,7 @@ export type PlatformWorkerFixtures = { }; export const platformTest = test.extend<{}, PlatformWorkerFixtures>({ - platform: [process.env.PLAYWRIGHT_DOCKER ? 'linux' : process.platform as 'win32' | 'darwin' | 'linux', { scope: 'worker' }], + platform: [process.platform as 'win32' | 'darwin' | 'linux', { scope: 'worker' }], isWindows: [process.platform === 'win32', { scope: 'worker' }], isMac: [process.platform === 'darwin', { scope: 'worker' }], isLinux: [process.platform === 'linux', { scope: 'worker' }], diff --git a/tests/installation/docker-integration.spec.ts b/tests/installation/docker-integration.spec.ts deleted file mode 100755 index 345fa0ba5b..0000000000 --- a/tests/installation/docker-integration.spec.ts +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { test, expect } from './npmTest'; -import * as path from 'path'; -import { TestServer } from '../../utils/testserver'; - - -// Skipping docker tests on CI on non-linux since GHA does not have -// Docker engine installed on macOS and Windows. -test.skip(() => process.env.CI && process.platform !== 'linux'); - -test.beforeAll(async ({ exec }) => { - // Delete any previous docker image to ensure clean run. - await exec('npx playwright docker delete-image', { - cwd: path.join(__dirname, '..', '..'), - }); -}); - -test('make sure it tells to run `npx playwright docker build` when image is not installed', async ({ exec }) => { - await exec('npm i --foreground-scripts @playwright/test'); - const result = await exec('npx playwright docker start', { - expectToExitWithError: true, - }); - expect(result).toContain('npx playwright docker build'); -}); - -test.describe('installed image', () => { - test.beforeAll(async ({ exec, daemonProcess, waitForPort }) => { - await exec('npx playwright docker build', { - env: { PWTEST_DOCKER_BASE_IMAGE: 'playwright:installation-tests-focal' }, - cwd: path.join(__dirname, '..', '..'), - }); - const dockerProcess = await daemonProcess({ - command: ['npx', 'playwright', 'docker', 'start', '--port=5667'], - shell: true, - cwd: path.join(__dirname, '..', '..'), - }); - await dockerProcess.waitForOutput('Endpoint:'); - }); - - test.afterAll(async ({ exec }) => { - await exec('npx playwright docker delete-image', { - cwd: path.join(__dirname, '..', '..'), - }); - }); - - test('all browsers work headless', async ({ exec }) => { - await exec('npm i --foreground-scripts @playwright/test'); - const result = await exec('npx playwright test docker.spec.js --grep platform --browser all', { - env: { PLAYWRIGHT_DOCKER: '1' }, - }); - expect(result).toContain('@chromium Linux'); - expect(result).toContain('@webkit Linux'); - expect(result).toContain('@firefox Linux'); - }); - - test('all browsers work headed', async ({ exec }) => { - await exec('npm i --foreground-scripts @playwright/test'); - { - const result = await exec(`npx playwright test docker.spec.js --headed --grep userAgent --browser chromium`, { - env: { PLAYWRIGHT_DOCKER: '1' }, - }); - expect(result).toContain('@chromium'); - expect(result).not.toContain('Headless'); - expect(result).toContain(' Chrome/'); - } - { - const result = await exec(`npx playwright test docker.spec.js --headed --grep userAgent --browser webkit`, { - env: { PLAYWRIGHT_DOCKER: '1' }, - }); - expect(result).toContain('@webkit'); - expect(result).toContain(' Version/'); - } - { - const result = await exec(`npx playwright test docker.spec.js --headed --grep userAgent --browser firefox`, { - env: { PLAYWRIGHT_DOCKER: '1' }, - }); - expect(result).toContain('@firefox'); - expect(result).toContain(' Firefox/'); - } - }); - - test('port forwarding works', async ({ exec, tmpWorkspace }) => { - await exec('npm i --foreground-scripts @playwright/test'); - const TEST_PORT = 8425; - const server = await TestServer.create(tmpWorkspace, TEST_PORT); - server.setRoute('/', (request, response) => { - response.end('Hello from host'); - }); - const result = await exec('npx playwright test docker.spec.js --grep localhost --browser all', { - env: { - TEST_PORT: TEST_PORT + '', - PLAYWRIGHT_DOCKER: '1' - }, - }); - expect(result).toContain('@chromium Hello from host'); - expect(result).toContain('@webkit Hello from host'); - expect(result).toContain('@firefox Hello from host'); - }); -}); - diff --git a/tests/library/playwright.config.ts b/tests/library/playwright.config.ts index 21e2f310e5..37f8eae7f2 100644 --- a/tests/library/playwright.config.ts +++ b/tests/library/playwright.config.ts @@ -34,11 +34,7 @@ const getExecutablePath = (browserName: BrowserName) => { return process.env.WKPATH; }; -let mode: TestModeName = 'default'; -if (process.env.PLAYWRIGHT_DOCKER) - mode = 'docker'; -else - mode = (process.env.PWTEST_MODE ?? 'default') as ('default' | 'driver' | 'service' | 'service2'); +const mode: TestModeName = (process.env.PWTEST_MODE ?? 'default') as ('default' | 'driver' | 'service' | 'service2'); const headed = process.argv.includes('--headed'); const channel = process.env.PWTEST_CHANNEL as any; const video = !!process.env.PWTEST_VIDEO; @@ -104,7 +100,6 @@ if (mode === 'service2') { metadata: { platform: process.platform, docker: !!process.env.INSIDE_DOCKER, - dockerIntegration: !!process.env.PLAYWRIGHT_DOCKER, headful: !!headed, browserName: 'chromium', channel, @@ -146,7 +141,6 @@ for (const browserName of browserNames) { metadata: { platform: process.platform, docker: !!process.env.INSIDE_DOCKER, - dockerIntegration: !!process.env.PLAYWRIGHT_DOCKER, headful: !!headed, browserName, channel, diff --git a/utils/docker/Dockerfile.vrt b/utils/docker/Dockerfile.vrt deleted file mode 100644 index ed3af615f6..0000000000 --- a/utils/docker/Dockerfile.vrt +++ /dev/null @@ -1,50 +0,0 @@ -FROM ubuntu:focal - -ARG DEBIAN_FRONTEND=noninteractive -ARG TZ=America/Los_Angeles -ARG DOCKER_IMAGE_NAME_TEMPLATE="mcr.microsoft.com/playwright:v%version%-vrt" - -# === INSTALL Node.js === - -RUN apt-get update && \ - # Install Node 18 - apt-get install -y curl wget gpg && \ - curl -sL https://deb.nodesource.com/setup_18.x | bash - && \ - apt-get install -y nodejs && \ - # Feature-parity with node.js base images. - apt-get install -y --no-install-recommends git openssh-client && \ - npm install -g yarn && \ - # clean apt cache - rm -rf /var/lib/apt/lists/* && \ - # Create the pwuser - adduser pwuser - -# === BAKE BROWSERS INTO IMAGE === - -ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright - -# 1. Add tip-of-tree Playwright package to install its browsers. -# The package should be built beforehand from tip-of-tree Playwright. -COPY ./playwright-core.tar.gz /tmp/playwright-core.tar.gz - -# 2. Bake in Playwright Agent. -# Playwright Agent is used to bake in browsers and browser dependencies, -# and run docker server later on. -# Browsers will be downloaded in `/ms-playwright`. -# Note: make sure to set 777 to the registry so that any user can access -# registry. -RUN mkdir /ms-playwright && \ - mkdir /ms-playwright-agent && \ - cd /ms-playwright-agent && npm init -y && \ - npm i /tmp/playwright-core.tar.gz && \ - npx playwright mark-docker-image "${DOCKER_IMAGE_NAME_TEMPLATE}" && \ - npx playwright install --with-deps && \ - npx playwright docker install-server-deps && \ - rm /tmp/playwright-core.tar.gz && \ - chmod -R 777 /ms-playwright && \ - rm -rf /var/lib/apt/lists/* - -WORKDIR /ms-playwright-agent -ENV DISPLAY_NUM=99 -ENV DISPLAY=:99 -ENTRYPOINT npx playwright docker run-server diff --git a/utils/docker/build.sh b/utils/docker/build.sh index c95ef42ec7..1d69a5d918 100755 --- a/utils/docker/build.sh +++ b/utils/docker/build.sh @@ -3,7 +3,7 @@ set -e set +x if [[ ($1 == '--help') || ($1 == '-h') || ($1 == '') || ($2 == '') ]]; then - echo "usage: $(basename $0) {--arm64,--amd64} {focal,jammy,vrt} playwright:localbuild-focal" + echo "usage: $(basename $0) {--arm64,--amd64} {focal,jammy} playwright:localbuild-focal" echo echo "Build Playwright docker image and tag it as 'playwright:localbuild-focal'." echo "Once image is built, you can run it with" diff --git a/utils/docker/publish_docker.sh b/utils/docker/publish_docker.sh index 0f32518622..3e78a338d5 100755 --- a/utils/docker/publish_docker.sh +++ b/utils/docker/publish_docker.sh @@ -53,15 +53,6 @@ if [[ "$RELEASE_CHANNEL" == "stable" ]]; then JAMMY_TAGS+=("jammy") fi -VRT_TAGS=( - "next-vrt" - "v${PW_VERSION}-vrt" -) - -if [[ "$RELEASE_CHANNEL" == "stable" ]]; then - VRT_TAGS+=("vrt") -fi - tag_and_push() { local source="$1" local target="$2" @@ -77,10 +68,8 @@ publish_docker_images_with_arch_suffix() { TAGS=("${FOCAL_TAGS[@]}") elif [[ "$FLAVOR" == "jammy" ]]; then TAGS=("${JAMMY_TAGS[@]}") - elif [[ "$FLAVOR" == "vrt" ]]; then - TAGS=("${VRT_TAGS[@]}") else - echo "ERROR: unknown flavor - $FLAVOR. Must be either 'focal', 'jammy' or 'vrt'" + echo "ERROR: unknown flavor - $FLAVOR. Must be either 'focal' or 'jammy'" exit 1 fi local ARCH="$2" @@ -105,10 +94,8 @@ publish_docker_manifest () { TAGS=("${FOCAL_TAGS[@]}") elif [[ "$FLAVOR" == "jammy" ]]; then TAGS=("${JAMMY_TAGS[@]}") - elif [[ "$FLAVOR" == "vrt" ]]; then - TAGS=("${VRT_TAGS[@]}") else - echo "ERROR: unknown flavor - $FLAVOR. Must be either 'focal', 'jammy' or 'vrt'" + echo "ERROR: unknown flavor - $FLAVOR. Must be either 'focal' or 'jammy'" exit 1 fi @@ -135,6 +122,3 @@ publish_docker_images_with_arch_suffix jammy amd64 publish_docker_images_with_arch_suffix jammy arm64 publish_docker_manifest jammy amd64 arm64 -publish_docker_images_with_arch_suffix vrt amd64 -publish_docker_images_with_arch_suffix vrt arm64 -publish_docker_manifest vrt amd64 arm64