devops: teach buildbots to report their status

browser_patches/buildbots/bots.md

@@ -3,6 +3,10 @@
 This document describes setting up bots infrastructure to produce
 browser builds.
 
+All bots require the following env variables to be set:
+- `AZ_ACCOUNT_KEY` and `AZ_ACCOUNT_NAME` secrets to be able to upload to PlayWright CDN
+- `TELEGRAM_BOT_KEY` report build progress to Telegram channel
+
 We currently have 4 build bots that produce 6 builds
 - **[buildbot-linux]** Ubuntu 18.04 machine
     - builds: `Webkit-Linux`, `Firefox-Linux`
@@ -24,8 +28,6 @@ Follow instructions on [Building Firefox for Windows](https://developer.mozilla.
 After this step, you should have `c:\mozilla-build` folder
 and `c:\mozilla-source` folder with firefox checkout.
 
-> **NOTE:** No spaces or quotes are allowed here!
-
 ### 2. Install azure-cli
 
 Install [azure-cli](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli-windows?view=azure-cli-latest) for windows using MS Installer
@@ -41,17 +43,20 @@ Run `cmd` as administrator and run the following line:
 This command will create a `c:\Windows\az` file that will call azure-cli with passed parameters (Not the most beautiful solution, but it works!)
 
 
-### 4. Provide CDN credentials to mingw env
+### 4. Set custom env variables to mingw env
 
-Edit `c:\mozilla-build\start-shell.bat` and add two lines in the beginning:
+Edit `c:\mozilla-build\start-shell.bat` and add the following lines in the beginning:
 
 ```bat
 SET AZ_ACCOUNT_NAME=<account-name>
 SET AZ_ACCOUNT_KEY=<account-key>
+SET TELEGRAM_BOT_KEY=<bot_key>
 ```
 
 change `<account-name>` and `<account-key>` with relevant keys/names.
 
+> **NOTE:** No spaces or quotes are allowed here!
+
 ### 5. Checkout PlayWright to /c/
 
 Run `c:\mozilla-build\start-shell.bat` and checkout PlayWright repo to `/c/playwright`.

browser_patches/buildbots/buildbot-linux.sh

@@ -41,6 +41,13 @@ mkdir -p $LOCKDIR
 trap "rm -rf ${LOCKDIR}; cd $(pwd -P); exit" INT TERM EXIT
 cd "$(dirname "$0")"
 
+IS_FIRST_RUN_FILE="/tmp/pw-buildbot-first-run.txt";
+if ! [[ -f $IS_FIRST_RUN_FILE ]]; then
+  touch "$IS_FIRST_RUN_FILE"
+  source ./send_telegram_message.sh
+  send_telegram_message '**Linux Buildbot Is Active**'
+fi
+
 # Check if git repo is dirty.
 if [[ -n $(git status -s) ]]; then
   echo "ERROR: dirty GIT state - commit everything and re-run the script."

browser_patches/buildbots/buildbot-mac-10.14.sh

@@ -47,6 +47,13 @@ mkdir -p $LOCKDIR
 trap "rm -rf ${LOCKDIR}; cd $(pwd -P); exit" INT TERM EXIT
 cd "$(dirname "$0")"
 
+IS_FIRST_RUN_FILE="/tmp/pw-buildbot-first-run.txt";
+if ! [[ -f $IS_FIRST_RUN_FILE ]]; then
+  touch "$IS_FIRST_RUN_FILE"
+  source ./send_telegram_message.sh
+  send_telegram_message '**Mac 10.14 Buildbot Is Active**'
+fi
+
 # Check if git repo is dirty.
 if [[ -n $(git status -s) ]]; then
   echo "ERROR: dirty GIT state - commit everything and re-run the script."

browser_patches/buildbots/buildbot-mac-10.15.sh

@@ -48,6 +48,13 @@ mkdir -p $LOCKDIR
 trap "rm -rf ${LOCKDIR}; cd $(pwd -P); exit" INT TERM EXIT
 cd "$(dirname "$0")"
 
+IS_FIRST_RUN_FILE="/tmp/pw-buildbot-first-run.txt";
+if ! [[ -f $IS_FIRST_RUN_FILE ]]; then
+  touch "$IS_FIRST_RUN_FILE"
+  source ./send_telegram_message.sh
+  send_telegram_message '**Mac 10.15 Buildbot Is Active**'
+fi
+
 # Check if git repo is dirty.
 if [[ -n $(git status -s) ]]; then
   echo "ERROR: dirty GIT state - commit everything and re-run the script."

browser_patches/buildbots/buildbot-windows.sh

@@ -36,6 +36,9 @@ if [[ -n $(git status -s) ]]; then
   exit 1
 fi
 
+source ./send_telegram_message.sh
+send_telegram_message '**Windows Buildbot Is Active**'
+
 iteration=0
 while true; do
   timestamp=$(date +%s)

browser_patches/buildbots/send_telegram_message.sh

@@ -0,0 +1,13 @@
+send_telegram_message() {
+  if [[ -z $TELEGRAM_BOT_KEY ]]; then
+    return;
+  fi
+  if ! command -v curl >/dev/null; then
+    return;
+  fi
+  curl --silent \
+       -X POST \
+       -H 'Content-Type: application/json' \
+       -d '{"chat_id": "-1001225613794", "text": "'"$1"'", "disable_notification": false}' \
+       https://api.telegram.org/bot$TELEGRAM_BOT_KEY/sendMessage >/dev/null
+}

browser_patches/checkout_build_archive_upload.sh

@@ -47,6 +47,7 @@ fi
 # - make sure the lockfile is removed when we exit and then claim it
 trap "rm -rf ${ZIP_PATH}; cd $(pwd -P); exit" INT TERM EXIT
 cd "$(dirname "$0")"
+BUILD_NUMBER=$(cat ./$BROWSER_NAME/BUILD_NUMBER)
 
 # pull from upstream and check if a new build has to be uploaded.
 if ! [[ ($2 == '-f') || ($2 == '--force') ]]; then
@@ -60,6 +61,10 @@ else
   echo "Force-rebuilding the build."
 fi
 
+source ./buildbots/send_telegram_message.sh
+BUILD_ALIAS=$(./upload.sh $BROWSER_NAME --show-alias $FFOX_WIN64)
+send_telegram_message "starting to build $BUILD_ALIAS"
+
 echo "-- preparing checkout"
 ./prepare_checkout.sh $BROWSER_NAME
 
@@ -74,10 +79,10 @@ echo "-- cleaning"
 ./$BROWSER_NAME/clean.sh
 
 echo "-- building"
-if [[ $BROWSER_NAME == "firefox" ]]; then
+if ./$BROWSER_NAME/build.sh $FFOX_WIN64; then
-  ./$BROWSER_NAME/build.sh $FFOX_WIN64
 else
-  ./$BROWSER_NAME/build.sh
+  send_telegram_message "$BUILD_ALIAS COMPILATION FAILED! ❌"
+  exit 1
 fi
 
 echo "-- archiving to $ZIP_PATH"
@@ -85,3 +90,4 @@ echo "-- archiving to $ZIP_PATH"
 
 echo "-- uploading"
 ./upload.sh $BROWSER_NAME $ZIP_PATH $FFOX_WIN64
+send_telegram_message "$BUILD_ALIAS uploaded ✅"

browser_patches/upload.sh

@@ -33,6 +33,7 @@ fi
 BROWSER_NAME=""
 BUILD_NUMBER=""
 BLOB_NAME=""
+ALIAS=""
 if [[ ("$1" == "firefox") || ("$1" == "firefox/") ]]; then
   # we always apply our patches atop of beta since it seems to get better
   # reliability guarantees.
@@ -40,13 +41,17 @@ if [[ ("$1" == "firefox") || ("$1" == "firefox/") ]]; then
   BROWSER_NAME="firefox"
   if [[ "$(uname)" == "Darwin" ]]; then
     BLOB_NAME="firefox-mac.zip"
+    ALIAS="firefox-mac r$BUILD_NUMBER"
   elif [[ "$(uname)" == "Linux" ]]; then
     BLOB_NAME="firefox-linux.zip"
+    ALIAS="ff-linux r$BUILD_NUMBER"
   elif [[ "$(uname)" == MINGW* ]]; then
     if [[ ("$2" == "--win64") || ("$3" == "--win64") ]]; then
       BLOB_NAME="firefox-win64.zip"
+      ALIAS="ff-win64 r$BUILD_NUMBER"
     else
       BLOB_NAME="firefox-win32.zip"
+      ALIAS="ff-win32 r$BUILD_NUMBER"
     fi
   else
     echo "ERROR: unzupported platform - $(uname)"
@@ -58,8 +63,10 @@ elif [[ ("$1" == "webkit") || ("$1" == "webkit/") ]]; then
   if [[ "$(uname)" == "Darwin" ]]; then
     MAC_MAJOR_MINOR_VERSION=$(sw_vers -productVersion | grep -o '^\d\+.\d\+')
     BLOB_NAME="minibrowser-mac-$MAC_MAJOR_MINOR_VERSION.zip"
+    ALIAS="webkit-mac-$MAC_MAJOR_MINOR_VERSION r$BUILD_NUMBER"
   elif [[ "$(uname)" == "Linux" ]]; then
     BLOB_NAME="minibrowser-linux.zip"
+    ALIAS="webkit-linux r$BUILD_NUMBER"
   else
     echo "ERROR: unzupported platform - $(uname)"
     exit 1
@@ -69,8 +76,13 @@ else
   exit 1
 fi
 
+if [[ ("$2" == '--show-alias') || ("$3" == '--show-alias') ]]; then
+  echo $ALIAS
+  exit 0
+fi
+
 BLOB_PATH="$BROWSER_NAME/$BUILD_NUMBER/$BLOB_NAME"
-if [[ $2 == '--check' ]]; then
+if [[ ("$2" == '--check') || ("$3" == '--check') ]]; then
   EXISTS=$(az storage blob exists -c builds --account-key $AZ_ACCOUNT_KEY --account-name $AZ_ACCOUNT_NAME -n "$BLOB_PATH" --query "exists")
   if [[ $EXISTS == "true" ]]; then
     exit 0