From 8dc740570afb1501a3cae592b3303e61efa5198f Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Tue, 16 Mar 2021 01:14:45 -0700 Subject: [PATCH] devops: refactor check_cdn.sh script (#5835) Introduce `EXPECTED_BUILDS` list in each application folder and use these lists when determining if all builds are ready. --- browser_patches/check_cdn.sh | 41 +++++ .../checkout_build_archive_upload.sh | 2 +- browser_patches/chromium/EXPECTED_BUILDS | 5 + .../EXPECTED_BUILDS | 2 + browser_patches/ffmpeg/EXPECTED_BUILDS | 5 + browser_patches/firefox/EXPECTED_BUILDS | 4 + browser_patches/tools/check_cdn.sh | 164 ------------------ browser_patches/webkit/EXPECTED_BUILDS | 6 + browser_patches/winldd/EXPECTED_BUILDS | 1 + 9 files changed, 65 insertions(+), 165 deletions(-) create mode 100755 browser_patches/check_cdn.sh create mode 100644 browser_patches/chromium/EXPECTED_BUILDS create mode 100644 browser_patches/deprecated-webkit-mac-10.14/EXPECTED_BUILDS create mode 100644 browser_patches/ffmpeg/EXPECTED_BUILDS create mode 100644 browser_patches/firefox/EXPECTED_BUILDS delete mode 100755 browser_patches/tools/check_cdn.sh create mode 100644 browser_patches/webkit/EXPECTED_BUILDS create mode 100644 browser_patches/winldd/EXPECTED_BUILDS diff --git a/browser_patches/check_cdn.sh b/browser_patches/check_cdn.sh new file mode 100755 index 0000000000..8973353808 --- /dev/null +++ b/browser_patches/check_cdn.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -e +set +x + +if [[ ($1 == '--help') || ($1 == '-h') ]]; then + echo "usage: $(basename $0) [firefox|webkit|chromium|ffmpeg] [--full-history] [--has-all-builds]" + echo + echo "List CDN status for browser" + echo + exit 0 +fi + +if [[ $# == 0 ]]; then + echo "missing browser: 'firefox', 'webkit', 'chromium' or 'ffmpeg'" + echo "try './$(basename $0) --help' for more information" + exit 1 +fi + +trap "cd $(pwd -P)" EXIT +cd "$(dirname "$0")" + +HOST="https://playwright2.blob.core.windows.net/builds" + +BROWSER_NAME="$1" +if [[ (! -f "./${BROWSER_NAME}/BUILD_NUMBER") || (! -f "./${BROWSER_NAME}/EXPECTED_BUILDS") ]]; then + echo ERROR: unknown application - "$1" + exit 1 +fi + +REVISION=$(head -1 "./${BROWSER_NAME}/BUILD_NUMBER") +BUILD_NAMES="./${BROWSER_NAME}/EXPECTED_BUILDS" + +for i in $(cat "${BUILD_NAMES}"); do + URL="${HOST}/${BROWSER_NAME}/${REVISION}/$i" + if ! [[ $(curl -s -L -I $URL | head -1 | cut -f2 -d' ') == 200 ]]; then + echo "${BROWSER_NAME} r${REVISION} is missing build: $i" + exit 1 + fi +done; + +echo "All expected builds are uploaded." diff --git a/browser_patches/checkout_build_archive_upload.sh b/browser_patches/checkout_build_archive_upload.sh index 05ed2fc6db..532ddce595 100755 --- a/browser_patches/checkout_build_archive_upload.sh +++ b/browser_patches/checkout_build_archive_upload.sh @@ -323,7 +323,7 @@ if generate_and_upload_browser_build 2>&1 | ./sanitize_and_compress_log.js $LOG_ send_telegram_message "$BUILD_ALIAS -- ${UPLOAD_SIZE}uploaded" # Check if we uploaded the last build. - if ./tools/check_cdn.sh $BROWSER_NAME --has-all-builds; then + if ./check_cdn.sh $BROWSER_NAME > /dev/null; then LAST_COMMIT_MESSAGE=$(git log --format=%s -n 1 HEAD -- ./$BROWSER_NAME/BUILD_NUMBER) send_telegram_message "$BROWSER_NAME r${BUILD_NUMBER} COMPLETE! ✅ $LAST_COMMIT_MESSAGE" fi diff --git a/browser_patches/chromium/EXPECTED_BUILDS b/browser_patches/chromium/EXPECTED_BUILDS new file mode 100644 index 0000000000..1c7a475624 --- /dev/null +++ b/browser_patches/chromium/EXPECTED_BUILDS @@ -0,0 +1,5 @@ +chromium-mac.zip +chromium-mac-arm64.zip +chromium-linux.zip +chromium-win32.zip +chromium-win64.zip diff --git a/browser_patches/deprecated-webkit-mac-10.14/EXPECTED_BUILDS b/browser_patches/deprecated-webkit-mac-10.14/EXPECTED_BUILDS new file mode 100644 index 0000000000..e1d6b231bf --- /dev/null +++ b/browser_patches/deprecated-webkit-mac-10.14/EXPECTED_BUILDS @@ -0,0 +1,2 @@ +deprecated-webkit-mac-10.14.zip + diff --git a/browser_patches/ffmpeg/EXPECTED_BUILDS b/browser_patches/ffmpeg/EXPECTED_BUILDS new file mode 100644 index 0000000000..309c97f502 --- /dev/null +++ b/browser_patches/ffmpeg/EXPECTED_BUILDS @@ -0,0 +1,5 @@ +ffmpeg-mac.zip +ffmpeg-linux.zip +ffmpeg-win32.zip +ffmpeg-win64.zip + diff --git a/browser_patches/firefox/EXPECTED_BUILDS b/browser_patches/firefox/EXPECTED_BUILDS new file mode 100644 index 0000000000..ce59da0dda --- /dev/null +++ b/browser_patches/firefox/EXPECTED_BUILDS @@ -0,0 +1,4 @@ +firefox-mac-10.14.zip +firefox-ubuntu-18.04.zip +firefox-win32.zip +firefox-win64.zip diff --git a/browser_patches/tools/check_cdn.sh b/browser_patches/tools/check_cdn.sh deleted file mode 100755 index d4dac601cc..0000000000 --- a/browser_patches/tools/check_cdn.sh +++ /dev/null @@ -1,164 +0,0 @@ -#!/bin/bash -set -e -set +x - -if [[ ($1 == '--help') || ($1 == '-h') ]]; then - echo "usage: $(basename $0) [firefox|webkit|chromium|ffmpeg] [--full-history] [--has-all-builds]" - echo - echo "List CDN status for browser" - echo - exit 0 -fi - -if [[ $# == 0 ]]; then - echo "missing browser: 'firefox', 'webkit', 'chromium' or 'ffmpeg'" - echo "try './$(basename $0) --help' for more information" - exit 1 -fi - -trap "cd $(pwd -P)" EXIT -cd "$(dirname "$0")" - -HOST="https://playwright2.blob.core.windows.net/builds" - -FFOX_REVISION=$(head -1 ../firefox/BUILD_NUMBER) -FFOX_ARCHIVES=( - "$HOST/firefox/%s/firefox-mac-10.14.zip" - "$HOST/firefox/%s/firefox-ubuntu-18.04.zip" - "$HOST/firefox/%s/firefox-win32.zip" - "$HOST/firefox/%s/firefox-win64.zip" -) -FFOX_ALIASES=( - "FF-MAC" - "FF-UBUNTU-18.04" - "FF-WIN32" - "FF-WIN64" -) - -WK_REVISION=$(head -1 ../webkit/BUILD_NUMBER) -WK_ARCHIVES=( - "$HOST/webkit/%s/webkit-ubuntu-18.04.zip" - "$HOST/webkit/%s/webkit-ubuntu-20.04.zip" - "$HOST/webkit/%s/webkit-mac-10.14.zip" - "$HOST/webkit/%s/webkit-mac-10.15.zip" - "$HOST/webkit/%s/webkit-mac-11.0-arm64.zip" - "$HOST/webkit/%s/webkit-win64.zip" -) -WK_ALIASES=( - "WK-UBUNTU-18.04" - "WK-UBUNTU-20.04" - "WK-MAC-10.14" - "WK-MAC-10.15" - "WK-MAC-11.0-ARM64" - "WK-WIN64" -) - -CR_REVISION=$(head -1 ../chromium/BUILD_NUMBER) -CR_ARCHIVES=( - "$HOST/chromium/%s/chromium-mac.zip" - "$HOST/chromium/%s/chromium-mac-arm64.zip" - "$HOST/chromium/%s/chromium-linux.zip" - "$HOST/chromium/%s/chromium-win32.zip" - "$HOST/chromium/%s/chromium-win64.zip" -) -CR_ALIASES=( - "CR-MAC" - "CR-MAC-ARM64" - "CR-LINUX" - "CR-WIN32" - "CR-WIN64" -) - -FFMPEG_REVISION=$(head -1 ../ffmpeg/BUILD_NUMBER) -FFMPEG_ARCHIVES=( - "$HOST/ffmpeg/%s/ffmpeg-mac.zip" - "$HOST/ffmpeg/%s/ffmpeg-linux.zip" - "$HOST/ffmpeg/%s/ffmpeg-win32.zip" - "$HOST/ffmpeg/%s/ffmpeg-win64.zip" -) -FFMPEG_ALIASES=( - "FFMPEG-MAC" - "FFMPEG-LINUX" - "FFMPEG-WIN32" - "FFMPEG-WIN64" -) - -WINLDD_REVISION=$(head -1 ../winldd/BUILD_NUMBER) -WINLDD_ARCHIVES=( - "$HOST/winldd/%s/winldd-win64.zip" -) -WINLDD_ALIASES=( - "WINLDD-WIN64" -) - -COLUMN="%-18s" -# COLORS -RED=$'\e[1;31m' -GRN=$'\e[1;32m' -YEL=$'\e[1;33m' -END=$'\e[0m' - -REVISION="" -ARCHIVES="" -ALIASES="" -if [[ ("$1" == "firefox") || ("$1" == "firefox/") ]]; then - REVISION=$FFOX_REVISION - ARCHIVES=("${FFOX_ARCHIVES[@]}") - ALIASES=("${FFOX_ALIASES[@]}") -elif [[ ("$1" == "webkit") || ("$1" == "webkit/") ]]; then - REVISION=$WK_REVISION - ARCHIVES=("${WK_ARCHIVES[@]}") - ALIASES=("${WK_ALIASES[@]}") -elif [[ ("$1" == "chromium") || ("$1" == "chromium/") ]]; then - REVISION=$CR_REVISION - ARCHIVES=("${CR_ARCHIVES[@]}") - ALIASES=("${CR_ALIASES[@]}") -elif [[ ("$1" == "ffmpeg") || ("$1" == "ffmpeg/") ]]; then - REVISION=$FFMPEG_REVISION - ARCHIVES=("${FFMPEG_ARCHIVES[@]}") - ALIASES=("${FFMPEG_ALIASES[@]}") -elif [[ ("$1" == "winldd") || ("$1" == "winldd/") ]]; then - REVISION=$WINLDD_REVISION - ARCHIVES=("${WINLDD_ARCHIVES[@]}") - ALIASES=("${WINLDD_ALIASES[@]}") -else - echo ERROR: unknown application - "$1" - exit 1 -fi - -if [[ $* == *--has-all-builds ]]; then - for i in "${ARCHIVES[@]}"; do - URL=$(printf $i $REVISION) - if ! [[ $(curl -s -L -I $URL | head -1 | cut -f2 -d' ') == 200 ]]; then - exit 1 - fi - done; - exit 0 -fi - -STOP_REVISION=$((REVISION - 3)) -if [[ $* == *--full-history* ]]; then - STOP_REVISION=0 -fi - -printf "%7s" "" -for i in "${ALIASES[@]}"; do - printf $COLUMN $i -done -printf "\n" -while (( REVISION > $STOP_REVISION )); do - printf "%-7s" ${REVISION} - for i in "${ARCHIVES[@]}"; do - URL=$(printf $i $REVISION) - if [[ $(curl -s -L -I $URL | head -1 | cut -f2 -d' ') == 200 ]]; then - printf ${GRN}$COLUMN${END} "YES" - else - printf ${RED}$COLUMN${END} "NO" - fi - done; - echo - REVISION=$((REVISION - 1 )) - if [[ $REVISION == "999" ]]; then - REVISION=2 - fi -done; diff --git a/browser_patches/webkit/EXPECTED_BUILDS b/browser_patches/webkit/EXPECTED_BUILDS new file mode 100644 index 0000000000..1a546dd483 --- /dev/null +++ b/browser_patches/webkit/EXPECTED_BUILDS @@ -0,0 +1,6 @@ +webkit-ubuntu-18.04.zip +webkit-ubuntu-20.04.zip +webkit-mac-10.15.zip +webkit-mac-11.0-arm64.zip +webkit-win64.zip + diff --git a/browser_patches/winldd/EXPECTED_BUILDS b/browser_patches/winldd/EXPECTED_BUILDS new file mode 100644 index 0000000000..cdead62b4c --- /dev/null +++ b/browser_patches/winldd/EXPECTED_BUILDS @@ -0,0 +1 @@ +winldd-win64.zip