diff --git a/browser_patches/checkout_build_archive_upload.sh b/browser_patches/checkout_build_archive_upload.sh
index 4b42700ca9..1cafaefda7 100755
--- a/browser_patches/checkout_build_archive_upload.sh
+++ b/browser_patches/checkout_build_archive_upload.sh
@@ -30,46 +30,57 @@ BROWSER_NAME=""
EXTRA_BUILD_ARGS=""
EXTRA_ARCHIVE_ARGS=""
BUILD_FLAVOR="$1"
+BUILD_BLOB_NAME=""
EXPECTED_HOST_OS=""
EXPECTED_HOST_OS_VERSION=""
if [[ "$BUILD_FLAVOR" == "firefox-linux" ]]; then
BROWSER_NAME="firefox"
EXPECTED_HOST_OS="Linux"
+ BUILD_BLOB_NAME="firefox-linux.zip"
elif [[ "$BUILD_FLAVOR" == "firefox-mac" ]]; then
BROWSER_NAME="firefox"
EXPECTED_HOST_OS="Darwin"
EXPECTED_HOST_OS_VERSION="10.14"
+ BUILD_BLOB_NAME="firefox-mac.zip"
elif [[ "$BUILD_FLAVOR" == "firefox-win32" ]]; then
BROWSER_NAME="firefox"
EXPECTED_HOST_OS="MINGW"
+ BUILD_BLOB_NAME="firefox-win32.zip"
elif [[ "$BUILD_FLAVOR" == "firefox-win64" ]]; then
BROWSER_NAME="firefox"
EXTRA_BUILD_ARGS="--win64"
EXPECTED_HOST_OS="MINGW"
+ BUILD_BLOB_NAME="firefox-win64.zip"
elif [[ "$BUILD_FLAVOR" == "webkit-gtk" ]]; then
BROWSER_NAME="webkit"
EXTRA_BUILD_ARGS="--gtk"
EXTRA_ARCHIVE_ARGS="--gtk"
EXPECTED_HOST_OS="Linux"
+ BUILD_BLOB_NAME="minibrowser-gtk.zip"
elif [[ "$BUILD_FLAVOR" == "webkit-wpe" ]]; then
BROWSER_NAME="webkit"
EXTRA_BUILD_ARGS="--wpe"
EXTRA_ARCHIVE_ARGS="--wpe"
EXPECTED_HOST_OS="Linux"
+ BUILD_BLOB_NAME="minibrowser-wpe.zip"
elif [[ "$BUILD_FLAVOR" == "webkit-gtk-wpe" ]]; then
BROWSER_NAME="webkit"
EXPECTED_HOST_OS="Linux"
+ BUILD_BLOB_NAME="minibrowser-gtk-wpe.zip"
elif [[ "$BUILD_FLAVOR" == "webkit-win64" ]]; then
BROWSER_NAME="webkit"
EXPECTED_HOST_OS="MINGW"
+ BUILD_BLOB_NAME="minibrowser-win64.zip"
elif [[ "$BUILD_FLAVOR" == "webkit-mac-10.14" ]]; then
BROWSER_NAME="webkit"
EXPECTED_HOST_OS="Darwin"
EXPECTED_HOST_OS_VERSION="10.14"
+ BUILD_BLOB_NAME="minibrowser-mac-10.14.zip"
elif [[ "$BUILD_FLAVOR" == "webkit-mac-10.15" ]]; then
BROWSER_NAME="webkit"
EXPECTED_HOST_OS="Darwin"
EXPECTED_HOST_OS_VERSION="10.15"
+ BUILD_BLOB_NAME="minibrowser-mac-10.15.zip"
else
echo ERROR: unknown build flavor - "$BUILD_FLAVOR"
exit 1
@@ -91,79 +102,106 @@ fi
if [[ $(uname) == MINGW* ]]; then
ZIP_PATH="$PWD/archive-$BROWSER_NAME.zip"
+ LOG_PATH="$PWD/log-$BROWSER_NAME.zip"
else
ZIP_PATH="/tmp/archive-$BROWSER_NAME.zip"
+ LOG_PATH="/tmp/log-$BROWSER_NAME.zip"
fi
if [[ -f $ZIP_PATH ]]; then
echo "Archive $ZIP_PATH already exists - remove and re-run the script."
exit 1
fi
-trap "rm -rf ${ZIP_PATH}; cd $(pwd -P);" INT TERM EXIT
+trap "rm -rf ${ZIP_PATH}; rm -rf ${LOG_PATH}; cd $(pwd -P);" INT TERM EXIT
cd "$(dirname "$0")"
BUILD_NUMBER=$(cat ./$BROWSER_NAME/BUILD_NUMBER)
+BUILD_BLOB_PATH="${BROWSER_NAME}/${BUILD_NUMBER}/${BUILD_BLOB_NAME}"
+LOG_BLOB_PATH="${BROWSER_NAME}/${BUILD_NUMBER}/${BUILD_BLOB_NAME%.zip}.log.zip"
# pull from upstream and check if a new build has to be uploaded.
if ! [[ ($2 == '-f') || ($2 == '--force') ]]; then
- if ./upload.sh $BUILD_FLAVOR --check; then
+ if ./upload.sh "${BUILD_BLOB_PATH}" --check; then
echo "Build is already uploaded - no changes."
exit 0
- else
- echo "Build is missing - rebuilding"
+ elif ./upload.sh "${LOG_BLOB_PATH}" --check; then
+ echo "This build has already been attempted - skip building."
+ exit 0
fi
+ echo "Build is missing and has not been attempted - rebuilding"
else
echo "Force-rebuilding the build."
fi
-source ./buildbots/send_telegram_message.sh
-BUILD_ALIAS="$BUILD_FLAVOR r$BUILD_NUMBER"
-
-send_telegram_message "$BUILD_ALIAS -- started"
-
-if [[ "$BUILD_FLAVOR" == "webkit-gtk-wpe" ]]; then
- echo "-- combining binaries together"
- if ! ./webkit/download_gtk_and_wpe_and_zip_together.sh $ZIP_PATH; then
- send_telegram_message "$BUILD_ALIAS -- ./download_gtk_and_wpe_and_zip_together.sh failed! ❌"
- exit 1
+FAILED_STEP=""
+function generate_and_upload_browser_build {
+ # webkit-gtk-wpe is a special build doesn't need to be built.
+ if [[ "$BUILD_FLAVOR" == "webkit-gtk-wpe" ]]; then
+ echo "-- combining binaries together"
+ if ! ./webkit/download_gtk_and_wpe_and_zip_together.sh $ZIP_PATH; then
+ FAILED_STEP="./download_gtk_and_wpe_and_zip_together.sh"
+ return 1
+ fi
+ echo "-- uploading"
+ if ! ./upload.sh $BUILD_BLOB_PATH $ZIP_PATH; then
+ FAILED_STEP="./upload.sh "
+ return 1
+ fi
+ return 0
fi
-else
+
+ # Other browser flavors follow typical build flow.
echo "-- preparing checkout"
if ! ./prepare_checkout.sh $BROWSER_NAME; then
- send_telegram_message "$BUILD_ALIAS -- ./prepare_checkout.sh failed! ❌"
- exit 1
+ FAILED_STEP="./prepare_checkout.sh"
+ return 1
fi
echo "-- cleaning"
if ! ./$BROWSER_NAME/clean.sh; then
- send_telegram_message "$BUILD_ALIAS -- ./clean.sh failed! ❌"
- exit 1
+ FAILED_STEP="./clean.sh"
+ return 1
fi
echo "-- building"
if ! ./$BROWSER_NAME/build.sh "$EXTRA_BUILD_ARGS"; then
- send_telegram_message "$BUILD_ALIAS -- ./build.sh failed! ❌"
- exit 1
+ FAILED_STEP="./build.sh "
+ return 1
fi
echo "-- archiving to $ZIP_PATH"
if ! ./$BROWSER_NAME/archive.sh $ZIP_PATH "$EXTRA_ARCHIVE_ARGS"; then
- send_telegram_message "$BUILD_ALIAS -- ./archive.sh failed! ❌"
- exit 1
+ FAILED_STEP="./archive.sh "
+ return 1
fi
+
+ echo "-- uploading"
+ if ! ./upload.sh $BUILD_BLOB_PATH $ZIP_PATH; then
+ FAILED_STEP="./upload.sh "
+ return 1
+ fi
+ return 0
+}
+
+source ./buildbots/send_telegram_message.sh
+BUILD_ALIAS="$BUILD_FLAVOR r$BUILD_NUMBER"
+send_telegram_message "$BUILD_ALIAS -- started"
+
+if generate_and_upload_browser_build 2>&1 | ./sanitize_env.js | zip > $LOG_PATH; then
+ # Report successful build. Note: we don't know how to get zip size on MINGW.
+ if [[ $(uname) == MINGW* ]]; then
+ send_telegram_message "$BUILD_ALIAS -- uploaded"
+ else
+ UPLOAD_SIZE=$(du -h "$ZIP_PATH" | awk '{print $1}')
+ send_telegram_message "$BUILD_ALIAS -- $UPLOAD_SIZE uploaded"
+ fi
+ # Check if we uploaded the last build.
+ if ./tools/check_cdn.sh $BROWSER_NAME --has-all-builds; then
+ LAST_COMMIT_MESSAGE=$(git log --format=%s -n 1 HEAD -- ./$BROWSER_NAME/BUILD_NUMBER)
+ send_telegram_message "$BROWSER_NAME r${BUILD_NUMBER} COMPLETE! ✅ $LAST_COMMIT_MESSAGE"
+ fi
+else
+ # Upload logs only in case of failure and report failure.
+ ./upload.sh ${LOG_BLOB_PATH} ${LOG_PATH} || true
+ send_telegram_message "$BUILD_ALIAS -- ${FAILED_STEP} failed! ❌ see logs"
fi
-echo "-- uploading"
-if ! ./upload.sh $BUILD_FLAVOR $ZIP_PATH; then
- send_telegram_message "$BUILD_ALIAS -- ./upload.sh failed! ❌"
- exit 1
-fi
-UPLOAD_SIZE=$(du -h "$ZIP_PATH" | awk '{print $1}')
-send_telegram_message "$BUILD_ALIAS -- $UPLOAD_SIZE uploaded"
-
-if ./tools/check_cdn.sh $BROWSER_NAME --has-all-builds; then
- LAST_COMMIT_MESSAGE=$(git log --format=%s -n 1 HEAD -- ./$BROWSER_NAME/BUILD_NUMBER)
- send_telegram_message "$BROWSER_NAME r${BUILD_NUMBER} COMPLETE! ✅ $LAST_COMMIT_MESSAGE"
-fi
-
-
-
diff --git a/browser_patches/sanitize_env.js b/browser_patches/sanitize_env.js
new file mode 100755
index 0000000000..aea961dbf0
--- /dev/null
+++ b/browser_patches/sanitize_env.js
@@ -0,0 +1,66 @@
+#!/usr/bin/env node
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const fs = require('fs');
+const readline = require('readline');
+
+// These env variable values should be removed from logs no matter what.
+const BLOCKLIST_ENV_KEYS = new Set([
+ 'AZ_ACCOUNT_NAME',
+ 'AZ_ACCOUNT_KEY',
+ 'TELEGRAM_BOT_KEY',
+]);
+
+// These env variable values can stay in logs - they are harmless.
+const ALLOWLIST_ENV_KEYS = new Set([
+ 'SHELL',
+ 'TERM',
+ 'USER',
+ 'PWD',
+ 'EDITOR',
+ 'LANG',
+ 'HOME',
+ 'LOGNAME',
+ 'COLORTERM',
+ 'TMPDIR',
+]);
+
+const sanitizeEnv = Object.entries(process.env).filter(([key, value]) => {
+ if (BLOCKLIST_ENV_KEYS.has(key))
+ return true;
+ if (ALLOWLIST_ENV_KEYS.has(key))
+ return false;
+ // Sanitize all env variables that have `KEY` or `ACCOUNT` as a name.
+ if (key.toUpperCase().includes('KEY') || key.toUpperCase().includes('ACCOUNT'))
+ return true;
+ // We shouldn't try sanitizing env values that are too short.
+ if (value.trim().length < 7)
+ return false;
+ return true;
+});
+
+const rl = readline.createInterface({
+ input: process.stdin,
+ crlfDelay: Infinity,
+});
+
+rl.on('line', line => {
+ for (const [key, value] of sanitizeEnv)
+ line = line.split(value).join(`<${key}>`);
+ console.log(line);
+});
+
diff --git a/browser_patches/upload.sh b/browser_patches/upload.sh
index 38d54a2ea0..8550288507 100755
--- a/browser_patches/upload.sh
+++ b/browser_patches/upload.sh
@@ -6,12 +6,12 @@ trap "cd $(pwd -P)" EXIT
cd "$(dirname "$0")"
if [[ ($1 == '--help') || ($1 == '-h') ]]; then
- echo "usage: $(basename $0) [firefox-linux|firefox-win32|firefox-win64|webkit-gtk|webkit-wpe|webkit-gtk-wpe|webkit-win64|webkit-mac-10.14|webkit-mac-10.15] [--check] [zip-path]"
+ echo "usage: $(basename $0) [BLOB-PATH] [--check|ZIP-PATH]"
echo
- echo "Upload .zip as a browser build."
+ echo "Upload ZIP-PATH to BLOB-PATH in `builds` container."
echo
echo "--check pass |--check| as a second parameter instead of a zip-path to check for"
- echo " the build existing in the CDN"
+ echo " existance of BLOB-PATH"
echo
echo "NOTE: \$AZ_ACCOUNT_KEY (azure account name) and \$AZ_ACCOUNT_NAME (azure account name)"
echo "env variables are required to upload builds to CDN."
@@ -25,54 +25,16 @@ if [[ (-z $AZ_ACCOUNT_KEY) || (-z $AZ_ACCOUNT_NAME) ]]; then
exit 1
fi
-if [[ $# < 1 ]]; then
- echo "missing browser: 'firefox' or 'webkit'"
+if [[ $# < 2 ]]; then
+ echo "not enought arguments!"
echo "try '$(basename $0) --help' for more information"
exit 1
fi
-BUILD_FLAVOR="$1"
-BROWSER_NAME=""
-BLOB_NAME=""
-if [[ "$BUILD_FLAVOR" == "firefox-linux" ]]; then
- BROWSER_NAME="firefox"
- BLOB_NAME="firefox-linux.zip"
-elif [[ "$BUILD_FLAVOR" == "firefox-mac" ]]; then
- BROWSER_NAME="firefox"
- BLOB_NAME="firefox-mac.zip"
-elif [[ "$BUILD_FLAVOR" == "firefox-win32" ]]; then
- BROWSER_NAME="firefox"
- BLOB_NAME="firefox-win32.zip"
-elif [[ "$BUILD_FLAVOR" == "firefox-win64" ]]; then
- BROWSER_NAME="firefox"
- BLOB_NAME="firefox-win64.zip"
-elif [[ "$BUILD_FLAVOR" == "webkit-gtk" ]]; then
- BROWSER_NAME="webkit"
- BLOB_NAME="minibrowser-gtk.zip"
-elif [[ "$BUILD_FLAVOR" == "webkit-wpe" ]]; then
- BROWSER_NAME="webkit"
- BLOB_NAME="minibrowser-wpe.zip"
-elif [[ "$BUILD_FLAVOR" == "webkit-gtk-wpe" ]]; then
- BROWSER_NAME="webkit"
- BLOB_NAME="minibrowser-gtk-wpe.zip"
-elif [[ "$BUILD_FLAVOR" == "webkit-win64" ]]; then
- BROWSER_NAME="webkit"
- BLOB_NAME="minibrowser-win64.zip"
-elif [[ "$BUILD_FLAVOR" == "webkit-mac-10.14" ]]; then
- BROWSER_NAME="webkit"
- BLOB_NAME="minibrowser-mac-10.14.zip"
-elif [[ "$BUILD_FLAVOR" == "webkit-mac-10.15" ]]; then
- BROWSER_NAME="webkit"
- BLOB_NAME="minibrowser-mac-10.15.zip"
-else
- echo ERROR: unknown build flavor - "$BUILD_FLAVOR"
- exit 1
-fi
+BLOB_PATH="$1"
+ZIP_PATH="$2"
-BUILD_NUMBER=$(cat ./$BROWSER_NAME/BUILD_NUMBER)
-BLOB_PATH="$BROWSER_NAME/$BUILD_NUMBER/$BLOB_NAME"
-
-if [[ ("$2" == '--check') || ("$3" == '--check') ]]; then
+if [[ ("$2" == '--check') ]]; then
EXISTS=$(az storage blob exists -c builds --account-key $AZ_ACCOUNT_KEY --account-name $AZ_ACCOUNT_NAME -n "$BLOB_PATH" --query "exists")
if [[ $EXISTS == "true" ]]; then
exit 0
@@ -81,14 +43,6 @@ if [[ ("$2" == '--check') || ("$3" == '--check') ]]; then
fi
fi
-if [[ $# < 2 ]]; then
- echo "missing path to zip archive to upload"
- echo "try '$(basename $0) --help' for more information"
- exit 1
-fi
-
-ZIP_PATH="$2"
-
if ! [[ -f $ZIP_PATH ]]; then
echo "ERROR: $ZIP_PATH does not exist"
exit 1