devops: produce ffmpeg builds on bots (#3820)

This patch moves FFMPEG building to buildbots:
- `ffmpeg-mac.zip` is built on Mac 10.14 machine
- `ffmpeg-win32.zip` and `ffmpeg-win64.zip` are cross-compiled on
  Ubuntu 20.04 machine

All builds across the platforms share the same config:
- the same versions of `ffmpeg` and `libvpx`
- the same build configuration for both `ffmpeg` and `libvpx`

The config could be found in the `//browser_patches/ffmpeg/CONFIG.sh`.

The builds will be then copied manually and committed to the git
repository.

browser_patches/buildbots/README.md

@@ -240,9 +240,16 @@ $ sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-9
 $ sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-9 100
 ```
 
+2. FFMPEG cross-compilation requires Docker. Install docker and add `$USER` to docker for sudo-less docker access
+
+```sh
+$ sudo apt-get install -y docker.io # install docker
+$ sudo usermod -aG docker $USER # add user to docker group
+$ newgrp docker # activate group changes
+```
+
 > **NOTE**: Firefox build config can be checked official Firefox builds, navigating to `about:buildconfig` URL.
 
 To document precisely my steps to bring up bots:
 - [July 22, 2020: Setting up Ubuntu 18.04 buildbot on Azure](https://gist.github.com/aslushnikov/a4a3823b894888546e741899e69a1d8e)
 - [July 22, 2020: Setting up Ubuntu 20.04 buildbot on Azure](https://gist.github.com/aslushnikov/a0bd658b575022e198443f856b5185e7)
-

browser_patches/buildbots/buildbot-mac-10.14.sh

@@ -65,3 +65,6 @@ git pull origin master
 
 git pull origin master
 ../checkout_build_archive_upload.sh webkit-mac-10.14 >/tmp/$(basename $0)--webkit-mac-10.14.log || true
+
+git pull origin master
+../checkout_build_archive_upload.sh ffmpeg-mac >/tmp/$(basename $0)--ffmpeg-mac.log || true

browser_patches/buildbots/buildbot-ubuntu-20.04.sh

@@ -64,3 +64,9 @@ touch "$IS_FIRST_RUN_FILE"
 
 git pull origin master
 ../checkout_build_archive_upload.sh webkit-ubuntu-20.04 >/tmp/$(basename $0)--webkit.log || true
+
+git pull origin master
+../checkout_build_archive_upload.sh ffmpeg-cross-compile-win32 >/tmp/$(basename $0)--ffmpeg-cross-compile-win32.log || true
+
+git pull origin master
+../checkout_build_archive_upload.sh ffmpeg-cross-compile-win64 >/tmp/$(basename $0)--ffmpeg-cross-compile-win64.log || true

browser_patches/checkout_build_archive_upload.sh

@@ -36,7 +36,25 @@ BUILD_FLAVOR="$1"
 BUILD_BLOB_NAME=""
 EXPECTED_HOST_OS=""
 EXPECTED_HOST_OS_VERSION=""
-if [[ "$BUILD_FLAVOR" == "chromium-linux-mirror-to-cdn" ]]; then
+if [[ "$BUILD_FLAVOR" == "ffmpeg-mac" ]]; then
+  BROWSER_NAME="ffmpeg"
+  EXTRA_BUILD_ARGS="--mac"
+  EXPECTED_HOST_OS="Darwin"
+  EXPECTED_HOST_OS_VERSION="10.14"
+  BUILD_BLOB_NAME="ffmpeg-mac.zip"
+elif [[ "$BUILD_FLAVOR" == "ffmpeg-cross-compile-win32" ]]; then
+  BROWSER_NAME="ffmpeg"
+  EXTRA_BUILD_ARGS="--cross-compile-win32"
+  EXPECTED_HOST_OS="Ubuntu"
+  EXPECTED_HOST_OS_VERSION="20.04"
+  BUILD_BLOB_NAME="ffmpeg-win32.zip"
+elif [[ "$BUILD_FLAVOR" == "ffmpeg-cross-compile-win64" ]]; then
+  BROWSER_NAME="ffmpeg"
+  EXTRA_BUILD_ARGS="--cross-compile-win64"
+  EXPECTED_HOST_OS="Ubuntu"
+  EXPECTED_HOST_OS_VERSION="20.04"
+  BUILD_BLOB_NAME="ffmpeg-win64.zip"
+elif [[ "$BUILD_FLAVOR" == "chromium-linux-mirror-to-cdn" ]]; then
   BROWSER_NAME="chromium"
   EXTRA_BUILD_ARGS="--linux"
   EXPECTED_HOST_OS="Ubuntu"

browser_patches/ffmpeg/.gitignore

@@ -0,0 +1,2 @@
+build/
+output/

browser_patches/ffmpeg/BUILD_NUMBER

@@ -0,0 +1 @@
+1000

browser_patches/ffmpeg/CONFIG.sh

@@ -0,0 +1,46 @@
+# Copyright (c) Microsoft Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the 'License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LIBVPX_VERSION="v1.9.0"
+LIBVPX_CONFIG="--enable-static \
+               --disable-shared \
+               --disable-docs \
+               --disable-tools \
+               --disable-unit-tests \
+               --disable-examples"
+
+FFMPEG_VERSION="n4.3.1"
+FFMPEG_CONFIG="--enable-gpl \
+               --enable-version3 \
+               --disable-debug \
+               --disable-everything \
+               --enable-ffmpeg \
+               --enable-protocol=pipe \
+               --enable-protocol=file \
+               --enable-parser=mjpeg \
+               --enable-decoder=mjpeg \
+               --enable-demuxer=image2pipe \
+               --enable-filter=pad \
+               --enable-filter=crop \
+               --enable-filter=scale \
+               --enable-muxer=webm \
+               --enable-libvpx \
+               --enable-static \
+               --enable-encoder=libvpx_vp8 \
+               --disable-pthreads \
+               --disable-zlib \
+               --disable-iconv \
+               --disable-w32threads \
+               --disable-bzlib"
+

browser_patches/ffmpeg/README.md

@@ -0,0 +1,44 @@
+# Playwright and FFMPEG
+
+Playwright requires FFMPEG to produce screncast. Playwright relies on stock
+FFMPEG on Ubuntu, and bundles FFMPEG binaries for Mac and Windows.
+
+## Configuration
+
+We compile `libvpx` and `ffmpeg` only. Their source versions and build
+configurations are defined in [`//browser_patches/ffmpeg/CONFIG.sh`](./CONFIG.sh).
+
+## Building `ffmpeg-mac`
+
+Cross-compilation scripts are based on:
+- https://trac.ffmpeg.org/wiki/CompilationGuide/Generic
+- https://trac.ffmpeg.org/wiki/CompilationGuide/macOS
+
+Prerequisites:
+- Mac
+- xcode command line tools: `xcode-select --install`
+- [homebrew](https://brew.sh/)
+
+Building:
+
+```
+~/playwright$ ./browser_patches/ffmpeg/build.sh --mac
+```
+
+## Building `ffmpeg-win*`
+
+Cross-compilation scripts are based on:
+- https://trac.ffmpeg.org/wiki/CompilationGuide/Generic
+- https://trac.ffmpeg.org/wiki/CompilationGuide/CrossCompilingForWindows
+
+Prerequisites:
+- Mac or Linux
+- [Docker](https://www.docker.com/)
+
+Building:
+
+```
+~/playwright$ ./browser_patches/ffmpeg/build.sh --cross-compile-win32
+~/playwright$ ./browser_patches/ffmpeg/build.sh --cross-compile-win64
+```
+

browser_patches/ffmpeg/archive.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -e
+set +x
+
+if [[ ("$1" == "-h") || ("$1" == "--help") ]]; then
+  echo "usage: $(basename $0) [output-absolute-path]"
+  echo
+  echo "Generate distributable .zip archive from ./output folder that was previously built."
+  echo
+  exit 0
+fi
+
+ZIP_PATH=$1
+if [[ $ZIP_PATH != /* ]]; then
+  echo "ERROR: path $ZIP_PATH is not absolute"
+  exit 1
+fi
+if [[ $ZIP_PATH != *.zip ]]; then
+  echo "ERROR: path $ZIP_PATH must have .zip extension"
+  exit 1
+fi
+if [[ -f $ZIP_PATH ]]; then
+  echo "ERROR: path $ZIP_PATH exists; can't do anything."
+  exit 1
+fi
+if ! [[ -d $(dirname $ZIP_PATH) ]]; then
+  echo "ERROR: folder for path $($ZIP_PATH) does not exist."
+  exit 1
+fi
+
+trap "cd $(pwd -P)" EXIT
+cd "$(dirname $0)"
+
+cp output/ffmpeg.zip $ZIP_PATH

browser_patches/ffmpeg/build-mac.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the 'License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+function die() { echo "$@"; exit 1; }
+
+if [[ "$(uname)" != "Darwin" ]]; then
+  die "ERROR: this script is designed to be run on OSX. Can't run on $(uname)"
+fi
+
+trap "cd $(pwd -P)" EXIT
+cd "$(dirname $0)"
+
+source ./CONFIG.sh
+
+BUILDDIR="${PWD}/build"
+PREFIX="${BUILDDIR}/osx_prefix"
+OUTPUT_PATH="${PWD}/output/ffmpeg-mac"
+
+function build_libvpx {
+  cd "${BUILDDIR}"
+  git clone https://chromium.googlesource.com/webm/libvpx
+  cd libvpx
+  git checkout "${LIBVPX_VERSION}"
+  # Compile libvpx according to the docs:
+  # - https://chromium.googlesource.com/webm/libvpx/+/master/README
+  ./configure --prefix="${PREFIX}" ${LIBVPX_CONFIG}
+  make && make install
+}
+
+function build_ffmpeg {
+  cd "${BUILDDIR}"
+  git clone git://source.ffmpeg.org/ffmpeg.git
+  cd ffmpeg
+  git checkout "${FFMPEG_VERSION}"
+  export PKG_CONFIG_PATH="${PREFIX}/lib/pkgconfig"
+  # Prohibit pkg-config from using system installed libs.
+  export PKG_CONFIG_LIBDIR=
+
+  ./configure --pkg-config=pkg-config \
+              --pkg-config-flags="--static" \
+              --extra-cflags="-I/${PREFIX}/include" \
+              --extra-ldflags="-L/${PREFIX}/lib" \
+              --prefix="${PREFIX}" \
+              --bindir="${PWD}/bin" \
+              ${FFMPEG_CONFIG}
+  make && make install
+}
+
+REQUIERED_BUILD_TOOLS=("git" "make" "yasm" "pkg-config")
+missing_build_tools=()
+
+for dependency in ${REQUIERED_BUILD_TOOLS[@]}; do
+  if ! command -v "${dependency}" >/dev/null; then
+    missing_build_tools+=("${dependency}")
+  fi
+done
+
+if [[ ${#missing_build_tools[@]} != 0 ]]; then
+  die "ERROR: missing dependencies! Please run:    brew install ${missing_build_tools[@]}"
+fi
+
+# Cleanup
+set -x
+rm -rf "${BUILDDIR}"
+mkdir -p "${BUILDDIR}"
+
+build_libvpx
+build_ffmpeg
+
+# put resulting executable where we were asked to
+mkdir -p $(dirname "${OUTPUT_PATH}")
+cp "${BUILDDIR}/ffmpeg/bin/ffmpeg" "${OUTPUT_PATH}"
+strip "${OUTPUT_PATH}"

browser_patches/ffmpeg/build.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the 'License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set +x
+
+trap "cd $(pwd -P)" EXIT
+cd "$(dirname $0)"
+
+if [[ ("$1" == "-h") || ("$1" == "--help") ]]; then
+  echo "usage: $(basename $0) [--mac|--cross-compile-win32|--cross-compile-win64]"
+  echo
+  echo "Build ffmpeg for the given platform"
+  echo
+  exit 0
+fi
+
+if [[ -z "$1" ]]; then
+  echo "ERROR: expected build target. Run with --help for more info"
+  exit 1
+fi
+
+rm -rf ./output
+mkdir -p output
+
+if [[ "$1" == "--mac" ]]; then
+  bash ./build-mac.sh
+  cd output && zip ffmpeg.zip ffmpeg-mac
+elif [[ "$1" == --cross-compile-win* ]]; then
+  if ! command -v docker >/dev/null; then
+    echo "ERROR: docker is required for the script"
+    exit 1
+  fi
+
+  if [[ "$1" == "--cross-compile-win32" ]]; then
+    time docker run --init --rm -v"${PWD}":/host -it ubuntu:18.04 bash /host/crosscompile-from-linux-to-win.sh --win32 /host/output/ffmpeg-win32.exe
+    cd output && zip ffmpeg.zip ffmpeg-win32.exe
+  elif [[ "$1" == "--cross-compile-win64" ]]; then
+    time docker run --init --rm -v"${PWD}":/host -it ubuntu:18.04 bash /host/crosscompile-from-linux-to-win.sh --win64 /host/output/ffmpeg-win64.exe
+    cd output && zip ffmpeg.zip ffmpeg-win64.exe
+  else
+    echo "ERROR: unsupported platform - $1"
+    exit 1
+  fi
+else
+  echo "ERROR: unsupported platform - $1"
+  exit 1
+fi
+

browser_patches/ffmpeg/clean.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+set +x
+
+trap "cd $(pwd -P)" EXIT
+cd "$(dirname $0)"
+
+rm -rf output
+

browser_patches/ffmpeg/crosscompile-from-linux-to-win.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the 'License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+function die() { echo "$@"; exit 1; }
+
+
+PREFIX="${HOME}/prefix"
+TOOLCHAIN_PREFIX_32="/usr/bin/i686-w64-mingw32-"
+TOOLCHAIN_PREFIX_64="/usr/bin/x86_64-w64-mingw32-"
+
+arch=""
+toolchain_prefix=""
+
+if [[ "$(uname)" != "Linux" ]]; then
+  echo "ERROR: this script is designed to be run on Linux. Can't run on $(uname)"
+  exit 1
+fi
+
+if [[ "$1" == "--win32" ]]; then
+  arch="win32";
+  toolchain_prefix="${TOOLCHAIN_PREFIX_32}"
+elif [[ "$1" == "--win64" ]]; then
+  arch="win64";
+  toolchain_prefix="${TOOLCHAIN_PREFIX_64}"
+elif [[ -z "$1" ]]; then
+  die "ERROR: expect --win32 or --win64 as the first argument"
+else
+  die "ERROR: unknown arch '$1' - expected --win32 or --win64"
+fi
+
+output_path="$2"
+if [[ -z "${output_path}" ]]; then
+  die "ERROR: output path is not specified"
+elif [[ "${output_path}" != /* ]]; then
+  die "ERROR: output path ${output_path} is not absolute"
+elif ! [[ -d $(dirname "${output_path}") ]]; then
+  die "ERROR: folder for output path ${output_path} does not exist."
+fi
+
+function build_libvpx {
+  cd "${HOME}"
+  git clone https://chromium.googlesource.com/webm/libvpx
+  cd libvpx
+  git checkout "${LIBVPX_VERSION}"
+  # Cross-compiling libvpx according to the docs:
+  # - https://chromium.googlesource.com/webm/libvpx/+/master/README
+  local target=""
+  if [[ $arch == "win32" ]]; then
+    target="x86-win32-gcc";
+  elif [[ $arch == "win64" ]]; then
+    target="x86_64-win64-gcc";
+  else
+    die "ERROR: unsupported arch to compile libvpx - $arch"
+  fi
+  CROSS="${toolchain_prefix}" ./configure --prefix="${PREFIX}" --target="${target}" ${LIBVPX_CONFIG}
+  CROSS="${toolchain_prefix}" make && make install
+}
+
+function build_ffmpeg {
+  cd "${HOME}"
+  git clone git://source.ffmpeg.org/ffmpeg.git
+  cd ffmpeg
+  git checkout "${FFMPEG_VERSION}"
+  export PKG_CONFIG_PATH="${PREFIX}/lib/pkgconfig"
+  # Prohibit pkg-config from using linux system installed libs.
+  export PKG_CONFIG_LIBDIR=
+
+  local ffmpeg_arch=""
+  if [[ $arch == "win32" ]]; then
+    ffmpeg_arch="x86";
+  elif [[ $arch == "win64" ]]; then
+    ffmpeg_arch="x86_64";
+  else
+    die "ERROR: unsupported arch to compile ffmpeg - $arch"
+  fi
+  ./configure --arch="${ffmpeg_arch}" \
+            --target-os=mingw32 \
+            --cross-prefix="${toolchain_prefix}" \
+            --pkg-config=pkg-config \
+            --pkg-config-flags="--static" \
+            --extra-cflags="-I/${PREFIX}/include" \
+            --extra-ldflags="-L/${PREFIX}/lib -static" \
+            --prefix="${PREFIX}" \
+            --bindir="${PWD}/bin" \
+            ${FFMPEG_CONFIG}
+  make && make install
+}
+
+trap "cd $(pwd -P)" EXIT
+cd "$(dirname $0)"
+
+source ./CONFIG.sh
+
+apt-get update
+apt-get install -y mingw-w64 git make yasm pkg-config
+
+build_libvpx
+build_ffmpeg
+
+# put resulting executable where we were asked to
+cp "${HOME}/ffmpeg/bin/ffmpeg.exe" "${output_path}"
+${toolchain_prefix}strip "${output_path}"

browser_patches/prepare_checkout.sh

@@ -34,7 +34,10 @@ BUILD_NUMBER=""
 WEBKIT_EXTRA_FOLDER_PATH=""
 FIREFOX_EXTRA_FOLDER_PATH=""
 if [[ ("$1" == "chromium") || ("$1" == "chromium/") || ("$1" == "cr") ]]; then
-  echo "FYI: chromium checkout is not supported."
+  echo "FYI: chromium checkout is not supported. Use '//browser_patches/chromium/build.sh' instead"
+  exit 0
+elif [[ ("$1" == "ffmpeg") || ("$1" == "ffmpeg/") ]]; then
+  echo "FYI: ffmpeg checkout is not supported. Use '//browser_patches/ffmpeg/build.sh' instead"
   exit 0
 elif [[ ("$1" == "firefox") || ("$1" == "firefox/") || ("$1" == "ff") ]]; then
   FRIENDLY_CHECKOUT_PATH="//browser_patches/firefox/checkout";

browser_patches/tools/check_cdn.sh

@@ -3,7 +3,7 @@ set -e
 set +x
 
 if [[ ($1 == '--help') || ($1 == '-h') ]]; then
-  echo "usage: $(basename $0) [firefox|webkit] [--full-history] [--has-all-builds]"
+  echo "usage: $(basename $0) [firefox|webkit|chromium|ffmpeg] [--full-history] [--has-all-builds]"
   echo
   echo "List CDN status for browser"
   echo
@@ -11,7 +11,7 @@ if [[ ($1 == '--help') || ($1 == '-h') ]]; then
 fi
 
 if [[ $# == 0 ]]; then
-  echo "missing browser: 'firefox' or 'webkit'"
+  echo "missing browser: 'firefox', 'webkit', 'chromium' or 'ffmpeg'"
   echo "try './$(basename $0) --help' for more information"
   exit 1
 fi
@@ -65,6 +65,18 @@ CR_ALIASES=(
   "CR-WIN64"
 )
 
+FFMPEG_REVISION=$(head -1 ../ffmpeg/BUILD_NUMBER)
+FFMPEG_ARCHIVES=(
+  "$HOST/ffmpeg/%s/ffmpeg-mac.zip"
+  "$HOST/ffmpeg/%s/ffmpeg-win32.zip"
+  "$HOST/ffmpeg/%s/ffmpeg-win64.zip"
+)
+FFMPEG_ALIASES=(
+  "FFMPEG-MAC"
+  "FFMPEG-WIN32"
+  "FFMPEG-WIN64"
+)
+
 COLUMN="%-18s"
 # COLORS
 RED=$'\e[1;31m'
@@ -87,8 +99,12 @@ elif [[ ("$1" == "chromium") || ("$1" == "chromium/") ]]; then
   REVISION=$CR_REVISION
   ARCHIVES=("${CR_ARCHIVES[@]}")
   ALIASES=("${CR_ALIASES[@]}")
+elif [[ ("$1" == "ffmpeg") || ("$1" == "ffmpeg/") ]]; then
+  REVISION=$FFMPEG_REVISION
+  ARCHIVES=("${FFMPEG_ARCHIVES[@]}")
+  ALIASES=("${FFMPEG_ALIASES[@]}")
 else
-  echo ERROR: unknown browser - "$1"
+  echo ERROR: unknown application - "$1"
   exit 1
 fi