Filip/playwright
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

devops: revamp scripts

Browse source
This commit is contained in:
Andrey Lushnikov 2019-11-21 17:06:57 -08:00
parent 1e91257d6e
commit 98e05b66fd
12 changed files with 161 additions and 188 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
browser_patches/checkout_build_archive_upload.sh
View file

@ -52,9 +52,7 @@ else
fi fi
echo "-- preparing checkout" echo "-- preparing checkout"
# the "do_checkout" script asks if you want to reset existing branch. ./prepare_checkout.sh $BROWSER_NAME
# sure we do!
yes | ./do_checkout.sh $BROWSER_NAME
echo "-- building" echo "-- building"
./$BROWSER_NAME/build.sh ./$BROWSER_NAME/build.sh
echo "-- archiving to $ZIP_PATH" echo "-- archiving to $ZIP_PATH"

131
browser_patches/do_checkout.sh
View file

@ -1,131 +0,0 @@
#!/bin/bash
set -e
set +x
trap "cd $(pwd -P)" EXIT
cd "$(dirname "$0")"
if [[ ($1 == '--help') || ($1 == '-h') ]]; then
echo "usage: $(basename $0) [firefox|webkit]"
echo
echo "Produces a browser checkout ready to be built."
echo
exit 0
fi
if [[ $# == 0 ]]; then
echo "missing browser: 'firefox' or 'webkit'"
echo "try './$(basename $0) --help' for more information"
exit 1
fi
# FRIENDLY_CHECKOUT_PATH is used only for logging.
FRIENDLY_CHECKOUT_PATH="";
CHECKOUT_PATH=""
# Export path is where we put the patches and BASE_REVISION
REMOTE_URL=""
BASE_BRANCH=""
if [[ ("$1" == "firefox") || ("$1" == "firefox/") ]]; then
# we always apply our patches atop of beta since it seems to get better
# reliability guarantees.
BASE_BRANCH="beta"
FRIENDLY_CHECKOUT_PATH="//browser_patches/firefox/checkout";
CHECKOUT_PATH="$PWD/firefox/checkout"
REMOTE_URL="https://github.com/mozilla/gecko-dev"
elif [[ ("$1" == "webkit") || ("$1" == "webkit/") ]]; then
# webkit has only a master branch.
BASE_BRANCH="master"
FRIENDLY_CHECKOUT_PATH="//browser_patches/webkit/checkout";
CHECKOUT_PATH="$PWD/webkit/checkout"
REMOTE_URL=""
REMOTE_URL="https://github.com/webkit/webkit"
else
echo ERROR: unknown browser - "$1"
exit 1
fi
# if there's no checkout folder - checkout one.
if ! [[ -d $CHECKOUT_PATH ]]; then
echo "-- $FRIENDLY_CHECKOUT_PATH is missing - checking out.."
git clone --single-branch --branch $BASE_BRANCH $REMOTE_URL $CHECKOUT_PATH
else
echo "-- checking $FRIENDLY_CHECKOUT_PATH folder - OK"
fi
# if folder exists but not a git repository - bail out.
if ! [[ -d $CHECKOUT_PATH/.git ]]; then
echo "ERROR: $FRIENDLY_CHECKOUT_PATH is not a git repository! Remove it and re-run the script."
exit 1
else
echo "-- checking $FRIENDLY_CHECKOUT_PATH is a git repo - OK"
fi
# Switch to git repository.
cd $CHECKOUT_PATH
# Check if git repo is dirty.
if [[ -n $(git status -s) ]]; then
echo "ERROR: $FRIENDLY_CHECKOUT_PATH has dirty GIT state - commit everything and re-run the script."
exit 1
fi
if [[ $(git config --get remote.origin.url) == "$REMOTE_URL" ]]; then
echo "-- checking git origin url to point to $REMOTE_URL - OK";
else
echo "ERROR: git origin url DOES NOT point to $REMOTE_URL. Remove $FRIENDLY_CHECKOUT_PATH and re-run the script.";
exit 1
fi
# if there's no "BASE_BRANCH" branch - bail out.
if ! git show-ref --verify --quiet refs/heads/$BASE_BRANCH; then
echo "ERROR: $FRIENDLY_CHECKOUT_PATH/ does not have '$BASE_BRANCH' branch! Remove checkout/ and re-run the script."
exit 1
else
echo "-- checking $FRIENDLY_CHECKOUT_PATH has 'beta' branch - OK"
fi
if ! [[ -z $(git log --oneline origin/$BASE_BRANCH..$BASE_BRANCH) ]]; then
echo "ERROR: branch '$BASE_BRANCH' and branch 'origin/$BASE_BRANCH' have diverged - bailing out. Remove checkout/ and re-run the script."
exit 1;
else
echo "-- checking that $BASE_BRANCH and origin/$BASE_BRANCH are not diverged - OK"
fi
git checkout $BASE_BRANCH
git pull origin $BASE_BRANCH
PINNED_COMMIT=$(cat ../BASE_REVISION)
if ! git cat-file -e $PINNED_COMMIT^{commit}; then
echo "ERROR: $FRIENDLY_CHECKOUT_PATH/ does not include the BASE_REVISION (@$PINNED_COMMIT). Remove checkout/ and re-run the script."
exit 1
else
echo "-- checking $FRIENDLY_CHECKOUT_PATH repo has BASE_REVISION (@$PINNED_COMMIT) commit - OK"
fi
# If there's already a PWDEV branch than we should check if it's fine to reset all changes
# to it.
if git show-ref --verify --quiet refs/heads/pwdev; then
read -p "Do you want to reset 'PWDEV' branch? (ALL CHANGES WILL BE LOST) Y/n " -n 1 -r
echo
# if it's not fine to reset branch - bail out.
if ! [[ $REPLY =~ ^[Yy]$ ]]; then
echo "If you want to keep the branch, than I can't do much! Bailing out!"
exit 1
else
git checkout pwdev
git reset --hard $PINNED_COMMIT
echo "-- PWDEV now points to BASE_REVISION (@$PINNED_COMMIT)"
fi
else
# Otherwise just create a new branch.
git checkout -b pwdev
git reset --hard $PINNED_COMMIT
echo "-- created 'pwdev' branch that points to BASE_REVISION (@$PINNED_COMMIT)."
fi
echo "-- applying all patches"
git am ../patches/*
echo
echo
echo "DONE. Browser is ready to be built."

54
browser_patches/export.sh
View file

@ -5,10 +5,15 @@ set +x
trap "cd $(pwd -P)" EXIT trap "cd $(pwd -P)" EXIT
cd "$(dirname "$0")" cd "$(dirname "$0")"
REMOTE_BROWSER_UPSTREAM="browser_upstream"
BUILD_BRANCH="playwright-build"
if [[ ($1 == '--help') || ($1 == '-h') ]]; then if [[ ($1 == '--help') || ($1 == '-h') ]]; then
echo "usage: export.sh [firefox|webkit] [custom_checkout_path]" echo "usage: export.sh [firefox|webkit] [custom_checkout_path]"
echo echo
echo "Exports BASE_REVISION and patch from the checkout to browser folder." echo "Exports patch from the current branch of the checkout to browser folder."
echo "The checkout has to be 'prepared', meaning that 'prepare_checkout.sh' should be"
echo "run against it first."
echo echo
echo "You can optionally specify custom_checkout_path if you have browser checkout somewhere else" echo "You can optionally specify custom_checkout_path if you have browser checkout somewhere else"
echo "and wish to export patches from it." echo "and wish to export patches from it."
@ -25,22 +30,17 @@ fi
# FRIENDLY_CHECKOUT_PATH is used only for logging. # FRIENDLY_CHECKOUT_PATH is used only for logging.
FRIENDLY_CHECKOUT_PATH=""; FRIENDLY_CHECKOUT_PATH="";
CHECKOUT_PATH="" CHECKOUT_PATH=""
# Export path is where we put the patches and BASE_REVISION
EXPORT_PATH="" EXPORT_PATH=""
BASE_BRANCH=""
if [[ ("$1" == "firefox") || ("$1" == "firefox/") ]]; then if [[ ("$1" == "firefox") || ("$1" == "firefox/") ]]; then
# we always apply our patches atop of beta since it seems to get better
# reliability guarantees.
BASE_BRANCH="origin/beta"
FRIENDLY_CHECKOUT_PATH="//browser_patches/firefox/checkout"; FRIENDLY_CHECKOUT_PATH="//browser_patches/firefox/checkout";
CHECKOUT_PATH="$PWD/firefox/checkout" CHECKOUT_PATH="$PWD/firefox/checkout"
EXPORT_PATH="$PWD/firefox/" EXPORT_PATH="$PWD/firefox/"
source "./firefox/UPSTREAM_CONFIG.sh"
elif [[ ("$1" == "webkit") || ("$1" == "webkit/") ]]; then elif [[ ("$1" == "webkit") || ("$1" == "webkit/") ]]; then
# webkit has only a master branch.
BASE_BRANCH="origin/master"
FRIENDLY_CHECKOUT_PATH="//browser_patches/webkit/checkout"; FRIENDLY_CHECKOUT_PATH="//browser_patches/webkit/checkout";
CHECKOUT_PATH="$PWD/webkit/checkout" CHECKOUT_PATH="$PWD/webkit/checkout"
EXPORT_PATH="$PWD/webkit/" EXPORT_PATH="$PWD/webkit/"
source "./webkit/UPSTREAM_CONFIG.sh"
else else
echo ERROR: unknown browser to export - "$1" echo ERROR: unknown browser to export - "$1"
exit 1 exit 1
@ -72,6 +72,17 @@ fi
# Switch to git repository. # Switch to git repository.
cd $CHECKOUT_PATH cd $CHECKOUT_PATH
# Setting up |$REMOTE_BROWSER_UPSTREAM| remote and fetch the $BASE_BRANCH
if git remote get-url $REMOTE_BROWSER_UPSTREAM >/dev/null; then
if ! [[ $(git remote get-url $REMOTE_BROWSER_UPSTREAM) == "$REMOTE_URL" ]]; then
echo "ERROR: remote $REMOTE_BROWSER_UPSTREAM is not pointng to '$REMOTE_URL'! run `prepare_checkout.sh` first"
exit 1
fi
else
echo "ERROR: checkout does not have $REMOTE_BROWSER_UPSTREAM; run `prepare_checkout.sh` first"
exit 1
fi
# Check if git repo is dirty. # Check if git repo is dirty.
if [[ -n $(git status -s) ]]; then if [[ -n $(git status -s) ]]; then
echo "ERROR: $FRIENDLY_CHECKOUT_PATH has dirty GIT state - aborting export." echo "ERROR: $FRIENDLY_CHECKOUT_PATH has dirty GIT state - aborting export."
@ -81,28 +92,29 @@ else
fi fi
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
MERGE_BASE=$(git merge-base $BASE_BRANCH $CURRENT_BRANCH) BASE_REVISION=$(git merge-base $REMOTE_BROWSER_UPSTREAM/$BASE_BRANCH $CURRENT_BRANCH)
echo "==============================================================" echo "=============================================================="
echo " Repository: $FRIENDLY_CHECKOUT_PATH" echo " Repository: $FRIENDLY_CHECKOUT_PATH"
echo " Changes between branches: $BASE_BRANCH..$CURRENT_BRANCH" echo " Changes between branches: $REMOTE_BROWSER_UPSTREAM/$BASE_BRANCH..$CURRENT_BRANCH"
echo " BASE_REVISION: $MERGE_BASE" echo " BASE_REVISION: $BASE_REIVSION"
echo echo
read -p "Export? Y/n " -n 1 -r
echo
# if it's not fine to reset branch - bail out.
if ! [[ $REPLY =~ ^[Yy]$ ]]; then
echo "Exiting."
exit 1
fi
echo $MERGE_BASE > $EXPORT_PATH/BASE_REVISION echo $BASE_REIVSION > $EXPORT_PATH/BASE_REVISION
git checkout -b tmpsquash_export_script $MERGE_BASE git checkout -b tmpsquash_export_script $BASE_REIVSION
git merge --squash $CURRENT_BRANCH git merge --squash $CURRENT_BRANCH
git commit -am "chore: bootstrap" if ! git commit -am "chore: bootstrap"; then
echo "No changes!"
git checkout $CURRENT_BRANCH
git branch -D tmpsquash_export_script
exit 0
fi
PATCH_NAME=$(git format-patch -1 HEAD) PATCH_NAME=$(git format-patch -1 HEAD)
mv $PATCH_NAME $EXPORT_PATH/patches/ mv $PATCH_NAME $EXPORT_PATH/patches/
git checkout $CURRENT_BRANCH git checkout $CURRENT_BRANCH
git branch -D tmpsquash_export_script git branch -D tmpsquash_export_script
echo "REMOTE_URL=\"$REMOTE_URL\"
BASE_BRANCH=\"$BASE_BRANCH\"
BASE_REVISION=\"$BASE_REIVSION\"" > $EXPORT_PATH/UPSTREAM_CONFIG.sh
# Increment BUILD_NUMBER # Increment BUILD_NUMBER
BUILD_NUMBER=$(cat $EXPORT_PATH/BUILD_NUMBER) BUILD_NUMBER=$(cat $EXPORT_PATH/BUILD_NUMBER)

1
browser_patches/firefox/BASE_REVISION
View file

@ -1 +0,0 @@
46ca28eadfe840021e2ea496fa6b26f924fa135b

22
browser_patches/firefox/README.md
View file

@ -1,22 +0,0 @@
# Building Juggler (Linux & Mac)
1. Run `./do_checkout.sh` script. This will create a "checkout" folder with gecko-dev mirror from
GitHub and apply the PlayWright-specific patches.
2. Run `./do_build.sh` script to compile browser. Note: you'll need to follow [build instructions](https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Build_Instructions) to setup host environment first.
# Updating `FIREFOX_REVISION` and `//patches/*`
The `./export.sh` script will export a patch that describes all the differences between the current branch in `./checkout`
and the `beta` branch in `./checkout`.
# Uploading to Azure CDN
Uploading requires having both `AZ_ACCOUNT_KEY` and `AZ_ACCOUNT_NAME` env variables to be defined.
The following sequence of steps will checkout, build and upload build to Azure CDN on both Linux and Mac:
```sh
$ ./do_checkout.sh
$ ./build.sh
$ ./upload.sh
```

3
browser_patches/firefox/UPSTREAM_CONFIG.sh Normal file
View file

@ -0,0 +1,3 @@
REMOTE_URL="https://github.com/mozilla/gecko-dev"
BASE_BRANCH="beta"
BASE_REVISION="46ca28eadfe840021e2ea496fa6b26f924fa135b"

8
browser_patches/firefox/build.sh
View file

@ -6,11 +6,13 @@ trap "cd $(pwd -P)" EXIT
cd "$(dirname $0)" cd "$(dirname $0)"
cd "checkout" cd "checkout"
if ! [[ $(git rev-parse --abbrev-ref HEAD) == "pwdev" ]]; then BUILD_BRANCH="playwright-build"
echo "ERROR: Cannot build any branch other than PWDEV"
if ! [[ $(git rev-parse --abbrev-ref HEAD) == "$BUILD_BRANCH" ]]; then
echo "ERROR: Cannot build any branch other than $BUILD_BRANCH"
exit 1; exit 1;
else else
echo "-- checking git branch is PWDEV - OK" echo "-- checking git branch is $BUILD_BRANCH - OK"
fi fi
if [[ "$(uname)" == "Darwin" ]]; then if [[ "$(uname)" == "Darwin" ]]; then

108
browser_patches/prepare_checkout.sh Executable file
View file

@ -0,0 +1,108 @@
#!/bin/bash
set -e
set +x
trap "cd $(pwd -P)" EXIT
cd "$(dirname "$0")"
REMOTE_BROWSER_UPSTREAM="browser_upstream"
BUILD_BRANCH="playwright-build"
if [[ ($1 == '--help') || ($1 == '-h') ]]; then
echo "usage: $(basename $0) [firefox|webkit] [custom_checkout_path]"
echo
echo "Prepares browser checkout. The checkout is a GIT repository that:"
echo "- has a '$REMOTE_BROWSER_UPSTREAM' remote pointing to a REMOTE_URL from UPSTREAM_CONFIG.sh"
echo "- has a '$BUILD_BRANCH' branch that is BASE_REVISION with all the patches applied."
echo
echo "You can optionally specify custom_checkout_path if you want to use some other browser checkout"
echo
exit 0
fi
if [[ $# == 0 ]]; then
echo "missing browser: 'firefox' or 'webkit'"
echo "try './$(basename $0) --help' for more information"
exit 1
fi
# FRIENDLY_CHECKOUT_PATH is used only for logging.
FRIENDLY_CHECKOUT_PATH="";
CHECKOUT_PATH=""
if [[ ("$1" == "firefox") || ("$1" == "firefox/") ]]; then
FRIENDLY_CHECKOUT_PATH="//browser_patches/firefox/checkout";
CHECKOUT_PATH="$PWD/firefox/checkout"
source "./firefox/UPSTREAM_CONFIG.sh"
elif [[ ("$1" == "webkit") || ("$1" == "webkit/") ]]; then
FRIENDLY_CHECKOUT_PATH="//browser_patches/webkit/checkout";
CHECKOUT_PATH="$PWD/webkit/checkout"
source "./webkit/UPSTREAM_CONFIG.sh"
else
echo ERROR: unknown browser - "$1"
exit 1
fi
# we will use this just for beauty.
if [[ $# == 2 ]]; then
echo "WARNING: using custom checkout path $CHECKOUT_PATH"
CHECKOUT_PATH=$2
FRIENDLY_CHECKOUT_PATH="<custom_checkout>"
fi
# if there's no checkout folder - checkout one.
if ! [[ -d $CHECKOUT_PATH ]]; then
echo "-- $FRIENDLY_CHECKOUT_PATH is missing - checking out.."
git clone --single-branch --branch $BASE_BRANCH $REMOTE_URL $CHECKOUT_PATH
else
echo "-- checking $FRIENDLY_CHECKOUT_PATH folder - OK"
fi
# if folder exists but not a git repository - bail out.
if ! [[ -d $CHECKOUT_PATH/.git ]]; then
echo "ERROR: $FRIENDLY_CHECKOUT_PATH is not a git repository! Remove it and re-run the script."
exit 1
else
echo "-- checking $FRIENDLY_CHECKOUT_PATH is a git repo - OK"
fi
# ============== SETTING UP GIT REPOSITORY ==============
cd $CHECKOUT_PATH
# Bail out if git repo is dirty.
if [[ -n $(git status -s) ]]; then
echo "ERROR: $FRIENDLY_CHECKOUT_PATH has dirty GIT state - commit everything and re-run the script."
exit 1
fi
# Setting up |$REMOTE_BROWSER_UPSTREAM| remote and fetch the $BASE_BRANCH
if git remote get-url $REMOTE_BROWSER_UPSTREAM >/dev/null; then
echo "-- setting |$REMOTE_BROWSER_UPSTREAM| remote url to $REMOTE_URL"
git remote set-url $REMOTE_BROWSER_UPSTREAM $REMOTE_URL
else
echo "-- adding |$REMOTE_BROWSER_UPSTREAM| remote to $REMOTE_URL"
git remote add $REMOTE_BROWSER_UPSTREAM $REMOTE_URL
fi
git fetch $REMOTE_BROWSER_UPSTREAM $BASE_BRANCH
# Check if we have the $BASE_REVISION commit in GIT
if ! git cat-file -e $BASE_REVISION^{commit}; then
echo "ERROR: $FRIENDLY_CHECKOUT_PATH/ does not include the BASE_REVISION (@$BASE_REVISION). Wrong revision number?"
exit 1
else
echo "-- checking $FRIENDLY_CHECKOUT_PATH repo has BASE_REVISION (@$BASE_REVISION) commit - OK"
fi
# Check out the $BASE_REVISION
git checkout $BASE_REVISION
# Create a playwright-build branch and apply all the patches to it.
if git show-ref --verify --quiet refs/heads/playwright-build; then
git branch -D playwright-build
fi
git checkout -b playwright-build
echo "-- applying all patches"
git am ../patches/*
echo
echo
echo "DONE. Browser is ready to be built."

6
browser_patches/tools/check_cdn.sh
View file

@ -17,17 +17,17 @@ ARCHIVES=(
"$HOST/firefox/%s/firefox-linux.zip" "$HOST/firefox/%s/firefox-linux.zip"
"$HOST/firefox/%s/firefox-win.zip" "$HOST/firefox/%s/firefox-win.zip"
"$HOST/webkit/%s/minibrowser-linux.zip" "$HOST/webkit/%s/minibrowser-linux.zip"
"$HOST/webkit/%s/minibrowser-mac10.14.zip" "$HOST/webkit/%s/minibrowser-mac-10.14.zip"
"$HOST/webkit/%s/minibrowser-mac10.15.zip" "$HOST/webkit/%s/minibrowser-mac-10.15.zip"
) )
ALIASES=( ALIASES=(
"FF-MAC" "FF-MAC"
"FF-LINUX" "FF-LINUX"
"FF-WIN" "FF-WIN"
"WK-LINUX"
"WK-MAC-10.14" "WK-MAC-10.14"
"WK-MAC-10.15" "WK-MAC-10.15"
"WK-LINUX"
) )
COLUMN="%-15s" COLUMN="%-15s"

1
browser_patches/webkit/BASE_REVISION
View file

@ -1 +0,0 @@
734d4fc1b6da79b72e234e4a50435f25dd90d769

3
browser_patches/webkit/UPSTREAM_CONFIG.sh Normal file
View file

@ -0,0 +1,3 @@
REMOTE_URL="https://github.com/webkit/webkit"
BASE_BRANCH="master"
BASE_REVISION="734d4fc1b6da79b72e234e4a50435f25dd90d769"

8
browser_patches/webkit/build.sh
View file

@ -6,11 +6,13 @@ trap "cd $(pwd -P)" EXIT
cd "$(dirname $0)" cd "$(dirname $0)"
cd "checkout" cd "checkout"
if ! [[ $(git rev-parse --abbrev-ref HEAD) == "pwdev" ]]; then BUILD_BRANCH="playwright-build"
echo "ERROR: Cannot build any branch other than PWDEV"
if ! [[ $(git rev-parse --abbrev-ref HEAD) == "$BUILD_BRANCH" ]]; then
echo "ERROR: Cannot build any branch other than $BUILD_BRANCH"
exit 1; exit 1;
else else
echo "-- checking git branch is PWDEV - OK" echo "-- checking git branch is $BUILD_BRANCH - OK"
fi fi
if [[ "$(uname)" == "Darwin" ]]; then if [[ "$(uname)" == "Darwin" ]]; then