devops: automation to compile chromium for mac arm64 (#5101)

References #5067

browser_patches/checkout_build_archive_upload.sh

@@ -66,27 +66,33 @@ elif [[ "$BUILD_FLAVOR" == "ffmpeg-cross-compile-win64" ]]; then
   EXPECTED_HOST_OS="Ubuntu"
   EXPECTED_HOST_OS_VERSION="20.04"
   BUILD_BLOB_NAME="ffmpeg-win64.zip"
+elif [[ "$BUILD_FLAVOR" == "chromium-mac-arm64" ]]; then
+  BROWSER_NAME="chromium"
+  EXTRA_BUILD_ARGS="--compile-mac-arm64"
+  EXPECTED_HOST_OS="Darwin"
+  EXPECTED_HOST_OS_VERSION="10.15"
+  BUILD_BLOB_NAME="chromium-mac-arm64.zip"
 elif [[ "$BUILD_FLAVOR" == "chromium-linux-mirror-to-cdn" ]]; then
   BROWSER_NAME="chromium"
-  EXTRA_BUILD_ARGS="--linux"
+  EXTRA_BUILD_ARGS="--mirror-linux"
   EXPECTED_HOST_OS="Ubuntu"
   EXPECTED_HOST_OS_VERSION="18.04"
   BUILD_BLOB_NAME="chromium-linux.zip"
 elif [[ "$BUILD_FLAVOR" == "chromium-mac-mirror-to-cdn" ]]; then
   BROWSER_NAME="chromium"
-  EXTRA_BUILD_ARGS="--mac"
+  EXTRA_BUILD_ARGS="--mirror-mac"
   EXPECTED_HOST_OS="Ubuntu"
   EXPECTED_HOST_OS_VERSION="18.04"
   BUILD_BLOB_NAME="chromium-mac.zip"
 elif [[ "$BUILD_FLAVOR" == "chromium-win32-mirror-to-cdn" ]]; then
   BROWSER_NAME="chromium"
-  EXTRA_BUILD_ARGS="--win32"
+  EXTRA_BUILD_ARGS="--mirror-win32"
   EXPECTED_HOST_OS="Ubuntu"
   EXPECTED_HOST_OS_VERSION="18.04"
   BUILD_BLOB_NAME="chromium-win32.zip"
 elif [[ "$BUILD_FLAVOR" == "chromium-win64-mirror-to-cdn" ]]; then
   BROWSER_NAME="chromium"
-  EXTRA_BUILD_ARGS="--win64"
+  EXTRA_BUILD_ARGS="--mirror-win64"
   EXPECTED_HOST_OS="Ubuntu"
   EXPECTED_HOST_OS_VERSION="18.04"
   BUILD_BLOB_NAME="chromium-win64.zip"

browser_patches/chromium/build.sh

@@ -5,56 +5,152 @@ set +x
 trap "cd $(pwd -P)" EXIT
 cd "$(dirname $0)"
 
-rm -rf output
+USAGE=$(cat<<EOF
-mkdir -p output
+  usage: $(basename $0) [--mirror|--mirror-linux|--mirror-win32|--mirror-win64|--mirror-mac|--compile-mac-arm64]
-cd output
 
-CRREV=$(head -1 ../BUILD_NUMBER)
+  Either compiles chromium or mirrors it from Chromium Continuous Builds CDN.
+EOF
+)
 
-CHROMIUM_URL=""
+SCRIPT_PATH=$(pwd -P)
-CHROMIUM_FOLDER_NAME=""
+CRREV=$(head -1 ./BUILD_NUMBER)
-CHROMIUM_FILES_TO_REMOVE=()
 
-PLATFORM="$1"
+main() {
-if [[ -z "${PLATFORM}" ]]; then
+  if [[ $1 == "--help" || $1 == "-h" ]]; then
-  CURRENT_HOST_OS="$(uname)"
+    echo "$USAGE"
-  if [[ "${CURRENT_HOST_OS}" == "Darwin" ]]; then
+    exit 0
-    PLATFORM="--mac"
+  elif [[ $1 == "--mirror"* ]]; then
-  elif [[ "${CURRENT_HOST_OS}" == "Linux" ]]; then
+    mirror_chromium $1
-    PLATFORM="--linux"
+  elif [[ $1 == "--compile"* ]]; then
-  elif [[ "${CURRENT_HOST_OS}" == MINGW* ]]; then
+    compile_chromium
-    PLATFORM="--win64"
   else
-    echo "ERROR: unsupported host platform - ${CURRENT_HOST_OS}"
+    echo "ERROR: unknown first argument. Use --help for details."
     exit 1
   fi
-fi
+}
 
-if [[ "${PLATFORM}" == "--win32" ]]; then
+compile_chromium() {
-  CHROMIUM_URL="https://storage.googleapis.com/chromium-browser-snapshots/Win/${CRREV}/chrome-win.zip"
+  if [[ -z "${CR_CHECKOUT_PATH}" ]]; then
-  CHROMIUM_FOLDER_NAME="chrome-win"
+    echo "ERROR: chromium compilation requires CR_CHECKOUT_PATH to be set to reuse checkout."
-  CHROMIUM_FILES_TO_REMOVE+=("chrome-win/interactive_ui_tests.exe")
+    exit 1
-elif [[ "${PLATFORM}" == "--win64" ]]; then
+  fi
-  CHROMIUM_URL="https://storage.googleapis.com/chromium-browser-snapshots/Win_x64/${CRREV}/chrome-win.zip"
+  if ! command -v gclient >/dev/null; then
-  CHROMIUM_FOLDER_NAME="chrome-win"
+    echo "ERROR: chromium compilation requires depot_tools to be installed!"
-  CHROMIUM_FILES_TO_REMOVE+=("chrome-win/interactive_ui_tests.exe")
+    exit 1
-elif [[ "${PLATFORM}" == "--mac" ]]; then
+  fi
-  CHROMIUM_URL="https://storage.googleapis.com/chromium-browser-snapshots/Mac/${CRREV}/chrome-mac.zip"
-  CHROMIUM_FOLDER_NAME="chrome-mac"
-elif [[ "${PLATFORM}" == "--linux" ]]; then
-  CHROMIUM_URL="https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/${CRREV}/chrome-linux.zip"
-  CHROMIUM_FOLDER_NAME="chrome-linux"
-else
-  echo "ERROR: unknown platform to build: $1"
-  exit 1
-fi
 
-echo "--> Pulling Chromium ${CRREV} for ${PLATFORM#--}"
+  CHROMIUM_FOLDER_NAME=""
+  CHROMIUM_FILES_TO_ARCHIVE=()
 
-curl --output chromium-upstream.zip "${CHROMIUM_URL}"
+  if [[ $1 == "--compile-mac-arm64" ]]; then
-unzip chromium-upstream.zip
+    # As of Jan, 2021 Chromium mac compilation requires Xcode12.2
-for file in ${CHROMIUM_FILES_TO_REMOVE[@]}; do
+    if [[ ! -d /Applications/Xcode12.2.app ]]; then
-  rm -f "${file}"
+      echo "ERROR: chromium mac arm64 compilation requires XCode 12.2 to be available"
-done
+      echo "in the Applications folder!"
+      exit 1
+    fi
+    # As of Jan, 2021 Chromium mac compilation is only possible on Intel macbooks.
+    # See https://chromium.googlesource.com/chromium/src.git/+/master/docs/mac_arm64.md
+    if [[ $(uname -m) != "x86_64" ]]; then
+      echo "ERROR: chromium mac arm64 compilation is (ironically) only supported on Intel Macbooks"
+      exit 1
+    fi
+    CHROMIUM_FOLDER_NAME="chrome-mac"
+    CHROMIUM_FILES_TO_ARCHIVE+=("Chromium.app")
+  fi
 
-zip --symlinks -r build.zip "${CHROMIUM_FOLDER_NAME}"
+  # Get chromium SHA from the build revision.
+  # This will get us the last redirect URL from the crrev.com service.
+  REVISION_URL=$(curl -ILs -o /dev/null -w %{url_effective} "https://crrev.com/${CRREV}")
+  CRSHA="${REVISION_URL##*/}"
+
+  # Update Chromium checkout. One might think that this step should go to `prepare_checkout.sh`
+  # script, but the `prepare_checkout.sh` is in fact designed to prepare a fork checkout, whereas
+  # we don't fork Chromium.
+  cd "${CR_CHECKOUT_PATH}/src"
+  git pull origin master
+  git checkout "${CRSHA}"
+  gclient sync
+
+  # Prepare build folder.
+  mkdir -p "./out/Default"
+  cat <<EOF>./out/Default/args.gn
+is_debug = false
+symbol_level = 0
+EOF
+
+  if [[ $1 == "--compile-mac-arm64" ]]; then
+    echo 'target_cpu = "arm64"' >> ./out/Default/args.gn
+  fi
+
+  # Compile Chromium.
+  gn gen out/Default
+  DEVELOPER_DIR=/Applications/Xcode12.2.app/Contents/Developer autoninja -C out/Default chrome
+
+  # Prepare resulting archive similarly to how we do it in mirror_chromium.
+  cd "$SCRIPT_PATH"
+  rm -rf output
+  mkdir -p "output/${CHROMIUM_FOLDER_NAME}"
+  for file in ${CHROMIUM_FILES_TO_ARCHIVE[@]}; do
+    ditto "${CR_CHECKOUT_PATH}/src/out/Default/${file}" "output/${CHROMIUM_FOLDER_NAME}/${file}"
+  done
+  cd output
+  zip --symlinks -r build.zip "${CHROMIUM_FOLDER_NAME}"
+}
+
+mirror_chromium() {
+  cd "$SCRIPT_PATH"
+  rm -rf output
+  mkdir -p output
+  cd output
+
+  CHROMIUM_URL=""
+  CHROMIUM_FOLDER_NAME=""
+  CHROMIUM_FILES_TO_REMOVE=()
+
+  PLATFORM="$1"
+  if [[ "${PLATFORM}" == "--mirror" ]]; then
+    CURRENT_HOST_OS="$(uname)"
+    if [[ "${CURRENT_HOST_OS}" == "Darwin" ]]; then
+      PLATFORM="--mirror-mac"
+    elif [[ "${CURRENT_HOST_OS}" == "Linux" ]]; then
+      PLATFORM="--mirror-linux"
+    elif [[ "${CURRENT_HOST_OS}" == MINGW* ]]; then
+      PLATFORM="--mirror-win64"
+    else
+      echo "ERROR: unsupported host platform - ${CURRENT_HOST_OS}"
+      exit 1
+    fi
+  fi
+
+  if [[ "${PLATFORM}" == "--mirror-win32" ]]; then
+    CHROMIUM_URL="https://storage.googleapis.com/chromium-browser-snapshots/Win/${CRREV}/chrome-win.zip"
+    CHROMIUM_FOLDER_NAME="chrome-win"
+    CHROMIUM_FILES_TO_REMOVE+=("chrome-win/interactive_ui_tests.exe")
+  elif [[ "${PLATFORM}" == "--mirror-win64" ]]; then
+    CHROMIUM_URL="https://storage.googleapis.com/chromium-browser-snapshots/Win_x64/${CRREV}/chrome-win.zip"
+    CHROMIUM_FOLDER_NAME="chrome-win"
+    CHROMIUM_FILES_TO_REMOVE+=("chrome-win/interactive_ui_tests.exe")
+  elif [[ "${PLATFORM}" == "--mirror-mac" ]]; then
+    CHROMIUM_URL="https://storage.googleapis.com/chromium-browser-snapshots/Mac/${CRREV}/chrome-mac.zip"
+    CHROMIUM_FOLDER_NAME="chrome-mac"
+  elif [[ "${PLATFORM}" == "--mirror-linux" ]]; then
+    CHROMIUM_URL="https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/${CRREV}/chrome-linux.zip"
+    CHROMIUM_FOLDER_NAME="chrome-linux"
+  else
+    echo "ERROR: unknown platform to build: $1"
+    exit 1
+  fi
+
+  echo "--> Pulling Chromium ${CRREV} for ${PLATFORM#--}"
+
+  curl --output chromium-upstream.zip "${CHROMIUM_URL}"
+  unzip chromium-upstream.zip
+  for file in ${CHROMIUM_FILES_TO_REMOVE[@]}; do
+    rm -f "${file}"
+  done
+
+  zip --symlinks -r build.zip "${CHROMIUM_FOLDER_NAME}"
+}
+
+main $1

browser_patches/chromium/clean.sh

@@ -6,4 +6,7 @@ trap "cd $(pwd -P)" EXIT
 cd "$(dirname $0)"
 
 rm -rf output
+if [[ ! -z "${CR_CHECKOUT_PATH}" ]]; then
+  rm -rf "${CR_CHECKOUT_PATH}/src/out"
+fi
 

browser_patches/tools/check_cdn.sh

@@ -56,12 +56,14 @@ WK_ALIASES=(
 CR_REVISION=$(head -1 ../chromium/BUILD_NUMBER)
 CR_ARCHIVES=(
   "$HOST/chromium/%s/chromium-mac.zip"
+  "$HOST/chromium/%s/chromium-mac-arm64.zip"
   "$HOST/chromium/%s/chromium-linux.zip"
   "$HOST/chromium/%s/chromium-win32.zip"
   "$HOST/chromium/%s/chromium-win64.zip"
 )
 CR_ALIASES=(
   "CR-MAC"
+  "CR-MAC-ARM64"
   "CR-LINUX"
   "CR-WIN32"
   "CR-WIN64"