devops: start uploading build logs to bots (#1870)

Logs will be saved next to the build archive, with the same name
and double extension `.log.zip`.

browser_patches/checkout_build_archive_upload.sh

@@ -30,46 +30,57 @@ BROWSER_NAME=""
 EXTRA_BUILD_ARGS=""
 EXTRA_ARCHIVE_ARGS=""
 BUILD_FLAVOR="$1"
+BUILD_BLOB_NAME=""
 EXPECTED_HOST_OS=""
 EXPECTED_HOST_OS_VERSION=""
 if [[ "$BUILD_FLAVOR" == "firefox-linux" ]]; then
   BROWSER_NAME="firefox"
   EXPECTED_HOST_OS="Linux"
+  BUILD_BLOB_NAME="firefox-linux.zip"
 elif [[ "$BUILD_FLAVOR" == "firefox-mac" ]]; then
   BROWSER_NAME="firefox"
   EXPECTED_HOST_OS="Darwin"
   EXPECTED_HOST_OS_VERSION="10.14"
+  BUILD_BLOB_NAME="firefox-mac.zip"
 elif [[ "$BUILD_FLAVOR" == "firefox-win32" ]]; then
   BROWSER_NAME="firefox"
   EXPECTED_HOST_OS="MINGW"
+  BUILD_BLOB_NAME="firefox-win32.zip"
 elif [[ "$BUILD_FLAVOR" == "firefox-win64" ]]; then
   BROWSER_NAME="firefox"
   EXTRA_BUILD_ARGS="--win64"
   EXPECTED_HOST_OS="MINGW"
+  BUILD_BLOB_NAME="firefox-win64.zip"
 elif [[ "$BUILD_FLAVOR" == "webkit-gtk" ]]; then
   BROWSER_NAME="webkit"
   EXTRA_BUILD_ARGS="--gtk"
   EXTRA_ARCHIVE_ARGS="--gtk"
   EXPECTED_HOST_OS="Linux"
+  BUILD_BLOB_NAME="minibrowser-gtk.zip"
 elif [[ "$BUILD_FLAVOR" == "webkit-wpe" ]]; then
   BROWSER_NAME="webkit"
   EXTRA_BUILD_ARGS="--wpe"
   EXTRA_ARCHIVE_ARGS="--wpe"
   EXPECTED_HOST_OS="Linux"
+  BUILD_BLOB_NAME="minibrowser-wpe.zip"
 elif [[ "$BUILD_FLAVOR" == "webkit-gtk-wpe" ]]; then
   BROWSER_NAME="webkit"
   EXPECTED_HOST_OS="Linux"
+  BUILD_BLOB_NAME="minibrowser-gtk-wpe.zip"
 elif [[ "$BUILD_FLAVOR" == "webkit-win64" ]]; then
   BROWSER_NAME="webkit"
   EXPECTED_HOST_OS="MINGW"
+  BUILD_BLOB_NAME="minibrowser-win64.zip"
 elif [[ "$BUILD_FLAVOR" == "webkit-mac-10.14" ]]; then
   BROWSER_NAME="webkit"
   EXPECTED_HOST_OS="Darwin"
   EXPECTED_HOST_OS_VERSION="10.14"
+  BUILD_BLOB_NAME="minibrowser-mac-10.14.zip"
 elif [[ "$BUILD_FLAVOR" == "webkit-mac-10.15" ]]; then
   BROWSER_NAME="webkit"
   EXPECTED_HOST_OS="Darwin"
   EXPECTED_HOST_OS_VERSION="10.15"
+  BUILD_BLOB_NAME="minibrowser-mac-10.15.zip"
 else
   echo ERROR: unknown build flavor - "$BUILD_FLAVOR"
   exit 1
@@ -91,79 +102,106 @@ fi
 
 if [[ $(uname) == MINGW* ]]; then
   ZIP_PATH="$PWD/archive-$BROWSER_NAME.zip"
+  LOG_PATH="$PWD/log-$BROWSER_NAME.zip"
 else
   ZIP_PATH="/tmp/archive-$BROWSER_NAME.zip"
+  LOG_PATH="/tmp/log-$BROWSER_NAME.zip"
 fi
 
 if [[ -f $ZIP_PATH ]]; then
   echo "Archive $ZIP_PATH already exists - remove and re-run the script."
   exit 1
 fi
-trap "rm -rf ${ZIP_PATH}; cd $(pwd -P);" INT TERM EXIT
+trap "rm -rf ${ZIP_PATH}; rm -rf ${LOG_PATH}; cd $(pwd -P);" INT TERM EXIT
 cd "$(dirname "$0")"
 BUILD_NUMBER=$(cat ./$BROWSER_NAME/BUILD_NUMBER)
+BUILD_BLOB_PATH="${BROWSER_NAME}/${BUILD_NUMBER}/${BUILD_BLOB_NAME}"
+LOG_BLOB_PATH="${BROWSER_NAME}/${BUILD_NUMBER}/${BUILD_BLOB_NAME%.zip}.log.zip"
 
 # pull from upstream and check if a new build has to be uploaded.
 if ! [[ ($2 == '-f') || ($2 == '--force') ]]; then
-  if ./upload.sh $BUILD_FLAVOR --check; then
+  if ./upload.sh "${BUILD_BLOB_PATH}" --check; then
     echo "Build is already uploaded - no changes."
     exit 0
-  else
+  elif ./upload.sh "${LOG_BLOB_PATH}" --check; then
-    echo "Build is missing - rebuilding"
+    echo "This build has already been attempted - skip building."
+    exit 0
   fi
+  echo "Build is missing and has not been attempted - rebuilding"
 else
   echo "Force-rebuilding the build."
 fi
 
-source ./buildbots/send_telegram_message.sh
+FAILED_STEP=""
-BUILD_ALIAS="$BUILD_FLAVOR r$BUILD_NUMBER"
+function generate_and_upload_browser_build {
-
+  # webkit-gtk-wpe is a special build doesn't need to be built.
-send_telegram_message "$BUILD_ALIAS -- started"
+  if [[ "$BUILD_FLAVOR" == "webkit-gtk-wpe" ]]; then
-
+    echo "-- combining binaries together"
-if [[ "$BUILD_FLAVOR" == "webkit-gtk-wpe" ]]; then
+    if ! ./webkit/download_gtk_and_wpe_and_zip_together.sh $ZIP_PATH; then
-  echo "-- combining binaries together"
+      FAILED_STEP="./download_gtk_and_wpe_and_zip_together.sh"
-  if ! ./webkit/download_gtk_and_wpe_and_zip_together.sh $ZIP_PATH; then
+      return 1
-    send_telegram_message "$BUILD_ALIAS -- ./download_gtk_and_wpe_and_zip_together.sh failed! ❌"
+    fi
-    exit 1
+    echo "-- uploading"
+    if ! ./upload.sh $BUILD_BLOB_PATH $ZIP_PATH; then
+      FAILED_STEP="./upload.sh "
+      return 1
+    fi
+    return 0
   fi
-else
+
+  # Other browser flavors follow typical build flow.
   echo "-- preparing checkout"
   if ! ./prepare_checkout.sh $BROWSER_NAME; then
-    send_telegram_message "$BUILD_ALIAS -- ./prepare_checkout.sh failed! ❌"
+    FAILED_STEP="./prepare_checkout.sh"
-    exit 1
+    return 1
   fi
 
   echo "-- cleaning"
   if ! ./$BROWSER_NAME/clean.sh; then
-    send_telegram_message "$BUILD_ALIAS -- ./clean.sh failed! ❌"
+    FAILED_STEP="./clean.sh"
-    exit 1
+    return 1
   fi
 
   echo "-- building"
   if ! ./$BROWSER_NAME/build.sh "$EXTRA_BUILD_ARGS"; then
-    send_telegram_message "$BUILD_ALIAS -- ./build.sh failed! ❌"
+    FAILED_STEP="./build.sh "
-    exit 1
+    return 1
   fi
 
   echo "-- archiving to $ZIP_PATH"
   if ! ./$BROWSER_NAME/archive.sh $ZIP_PATH "$EXTRA_ARCHIVE_ARGS"; then
-    send_telegram_message "$BUILD_ALIAS -- ./archive.sh failed! ❌"
+    FAILED_STEP="./archive.sh "
-    exit 1
+    return 1
   fi
+
+  echo "-- uploading"
+  if ! ./upload.sh $BUILD_BLOB_PATH $ZIP_PATH; then
+    FAILED_STEP="./upload.sh "
+    return 1
+  fi
+  return 0
+}
+
+source ./buildbots/send_telegram_message.sh
+BUILD_ALIAS="$BUILD_FLAVOR r$BUILD_NUMBER"
+send_telegram_message "$BUILD_ALIAS -- started"
+
+if generate_and_upload_browser_build 2>&1 | ./sanitize_env.js | zip > $LOG_PATH; then
+  # Report successful build. Note: we don't know how to get zip size on MINGW.
+  if [[ $(uname) == MINGW* ]]; then
+    send_telegram_message "$BUILD_ALIAS -- uploaded"
+  else
+    UPLOAD_SIZE=$(du -h "$ZIP_PATH" | awk '{print $1}')
+    send_telegram_message "$BUILD_ALIAS -- $UPLOAD_SIZE uploaded"
+  fi
+  # Check if we uploaded the last build.
+  if ./tools/check_cdn.sh $BROWSER_NAME --has-all-builds; then
+    LAST_COMMIT_MESSAGE=$(git log --format=%s -n 1 HEAD -- ./$BROWSER_NAME/BUILD_NUMBER)
+    send_telegram_message "<b>$BROWSER_NAME r${BUILD_NUMBER} COMPLETE! ✅</b> $LAST_COMMIT_MESSAGE"
+  fi
+else
+  # Upload logs only in case of failure and report failure.
+  ./upload.sh ${LOG_BLOB_PATH} ${LOG_PATH} || true
+  send_telegram_message "$BUILD_ALIAS -- ${FAILED_STEP} failed! ❌ <a href='https://playwright.azureedge.net/builds/${LOG_BLOB_PATH}'>see logs</a>"
 fi
 
-echo "-- uploading"
-if ! ./upload.sh $BUILD_FLAVOR $ZIP_PATH; then
-  send_telegram_message "$BUILD_ALIAS -- ./upload.sh failed! ❌"
-  exit 1
-fi
-UPLOAD_SIZE=$(du -h "$ZIP_PATH" | awk '{print $1}')
-send_telegram_message "$BUILD_ALIAS -- $UPLOAD_SIZE uploaded"
-
-if ./tools/check_cdn.sh $BROWSER_NAME --has-all-builds; then
-  LAST_COMMIT_MESSAGE=$(git log --format=%s -n 1 HEAD -- ./$BROWSER_NAME/BUILD_NUMBER)
-  send_telegram_message "<b>$BROWSER_NAME r${BUILD_NUMBER} COMPLETE! ✅</b> $LAST_COMMIT_MESSAGE"
-fi
-
-
-

browser_patches/sanitize_env.js

@@ -0,0 +1,66 @@
+#!/usr/bin/env node
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const fs = require('fs');
+const readline = require('readline');
+
+// These env variable values should be removed from logs no matter what.
+const BLOCKLIST_ENV_KEYS = new Set([
+  'AZ_ACCOUNT_NAME',
+  'AZ_ACCOUNT_KEY',
+  'TELEGRAM_BOT_KEY',
+]);
+
+// These env variable values can stay in logs - they are harmless.
+const ALLOWLIST_ENV_KEYS = new Set([
+  'SHELL',
+  'TERM',
+  'USER',
+  'PWD',
+  'EDITOR',
+  'LANG',
+  'HOME',
+  'LOGNAME',
+  'COLORTERM',
+  'TMPDIR',
+]);
+
+const sanitizeEnv = Object.entries(process.env).filter(([key, value]) => {
+  if (BLOCKLIST_ENV_KEYS.has(key))
+    return true;
+  if (ALLOWLIST_ENV_KEYS.has(key))
+    return false;
+  // Sanitize all env variables that have `KEY` or `ACCOUNT` as a name.
+  if (key.toUpperCase().includes('KEY') || key.toUpperCase().includes('ACCOUNT'))
+    return true;
+  // We shouldn't try sanitizing env values that are too short.
+  if (value.trim().length < 7)
+    return false;
+  return true;
+});
+
+const rl = readline.createInterface({
+  input: process.stdin,
+  crlfDelay: Infinity,
+});
+
+rl.on('line', line => {
+  for (const [key,  value] of sanitizeEnv)
+    line = line.split(value).join(`<${key}>`);
+  console.log(line);
+});
+

browser_patches/upload.sh

@@ -6,12 +6,12 @@ trap "cd $(pwd -P)" EXIT
 cd "$(dirname "$0")"
 
 if [[ ($1 == '--help') || ($1 == '-h') ]]; then
-  echo "usage: $(basename $0) [firefox-linux|firefox-win32|firefox-win64|webkit-gtk|webkit-wpe|webkit-gtk-wpe|webkit-win64|webkit-mac-10.14|webkit-mac-10.15] [--check] [zip-path]"
+  echo "usage: $(basename $0) [BLOB-PATH] [--check|ZIP-PATH]"
   echo
-  echo "Upload .zip as a browser build."
+  echo "Upload ZIP-PATH to BLOB-PATH in `builds` container."
   echo
   echo "--check      pass |--check| as a second parameter instead of a zip-path to check for"
-  echo "             the build existing in the CDN"
+  echo "             existance of BLOB-PATH"
   echo
   echo "NOTE: \$AZ_ACCOUNT_KEY (azure account name) and \$AZ_ACCOUNT_NAME (azure account name)"
   echo "env variables are required to upload builds to CDN."
@@ -25,54 +25,16 @@ if [[ (-z $AZ_ACCOUNT_KEY) || (-z $AZ_ACCOUNT_NAME) ]]; then
   exit 1
 fi
 
-if [[ $# < 1 ]]; then
+if [[ $# < 2 ]]; then
-  echo "missing browser: 'firefox' or 'webkit'"
+  echo "not enought arguments!"
   echo "try '$(basename $0) --help' for more information"
   exit 1
 fi
 
-BUILD_FLAVOR="$1"
+BLOB_PATH="$1"
-BROWSER_NAME=""
+ZIP_PATH="$2"
-BLOB_NAME=""
-if [[ "$BUILD_FLAVOR" == "firefox-linux" ]]; then
-  BROWSER_NAME="firefox"
-  BLOB_NAME="firefox-linux.zip"
-elif [[ "$BUILD_FLAVOR" == "firefox-mac" ]]; then
-  BROWSER_NAME="firefox"
-  BLOB_NAME="firefox-mac.zip"
-elif [[ "$BUILD_FLAVOR" == "firefox-win32" ]]; then
-  BROWSER_NAME="firefox"
-  BLOB_NAME="firefox-win32.zip"
-elif [[ "$BUILD_FLAVOR" == "firefox-win64" ]]; then
-  BROWSER_NAME="firefox"
-  BLOB_NAME="firefox-win64.zip"
-elif [[ "$BUILD_FLAVOR" == "webkit-gtk" ]]; then
-  BROWSER_NAME="webkit"
-  BLOB_NAME="minibrowser-gtk.zip"
-elif [[ "$BUILD_FLAVOR" == "webkit-wpe" ]]; then
-  BROWSER_NAME="webkit"
-  BLOB_NAME="minibrowser-wpe.zip"
-elif [[ "$BUILD_FLAVOR" == "webkit-gtk-wpe" ]]; then
-  BROWSER_NAME="webkit"
-  BLOB_NAME="minibrowser-gtk-wpe.zip"
-elif [[ "$BUILD_FLAVOR" == "webkit-win64" ]]; then
-  BROWSER_NAME="webkit"
-  BLOB_NAME="minibrowser-win64.zip"
-elif [[ "$BUILD_FLAVOR" == "webkit-mac-10.14" ]]; then
-  BROWSER_NAME="webkit"
-  BLOB_NAME="minibrowser-mac-10.14.zip"
-elif [[ "$BUILD_FLAVOR" == "webkit-mac-10.15" ]]; then
-  BROWSER_NAME="webkit"
-  BLOB_NAME="minibrowser-mac-10.15.zip"
-else
-  echo ERROR: unknown build flavor - "$BUILD_FLAVOR"
-  exit 1
-fi
 
-BUILD_NUMBER=$(cat ./$BROWSER_NAME/BUILD_NUMBER)
+if [[ ("$2" == '--check') ]]; then
-BLOB_PATH="$BROWSER_NAME/$BUILD_NUMBER/$BLOB_NAME"
-
-if [[ ("$2" == '--check') || ("$3" == '--check') ]]; then
   EXISTS=$(az storage blob exists -c builds --account-key $AZ_ACCOUNT_KEY --account-name $AZ_ACCOUNT_NAME -n "$BLOB_PATH" --query "exists")
   if [[ $EXISTS == "true" ]]; then
     exit 0
@@ -81,14 +43,6 @@ if [[ ("$2" == '--check') || ("$3" == '--check') ]]; then
   fi
 fi
 
-if [[ $# < 2 ]]; then
-  echo "missing path to zip archive to upload"
-  echo "try '$(basename $0) --help' for more information"
-  exit 1
-fi
-
-ZIP_PATH="$2"
-
 if ! [[ -f $ZIP_PATH ]]; then
   echo "ERROR: $ZIP_PATH does not exist"
   exit 1