devops: simplify building webkit on linux bots (#3127)

Originally there was no way to build both webkit-gtk and webkit-wpe
in the same checkout. As a result, we were:
- building webkit-gtk & uploading it to CDN
- building webkit-wpe & uploading it to CDN
- downloading webkit-gtk & webkit-wpe from CDN and compiling a single
webkit-gtk-wpe folder that we uploaded to CDN

As of today, however, we can build WPE and GTK ports together. This
patch starts using this to drastically simplify build process:
- build webkit-gtk and webkit-wpe
- use `archive.sh` script that compiles a single folder with both ports
- upload archive to the cloud

This should also fix currently failing webkit build that fails to
assemble the `webkit-gtk-wpe` folder (I missed these codepaths while preparing
for Ubuntu-20.04 bot).

References #2745
This commit is contained in:
Andrey Lushnikov 2020-07-23 15:57:53 -07:00 committed by GitHub
parent d234dac752
commit c1032ae483
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 88 additions and 264 deletions

View file

@ -66,6 +66,4 @@ git pull origin master
../checkout_build_archive_upload.sh firefox-ubuntu-18.04 >/tmp/$(basename $0)--firefox.log || true
git pull origin master
../checkout_build_archive_upload.sh webkit-gtk-ubuntu-18.04 >/tmp/$(basename $0)--webkit-gtk.log || true
../checkout_build_archive_upload.sh webkit-wpe-ubuntu-18.04 >/tmp/$(basename $0)--webkit-wpe.log || true
../checkout_build_archive_upload.sh webkit-gtk-wpe-ubuntu-18.04 >/tmp/$(basename $0)--webkit-gtk-wpe.log || true
../checkout_build_archive_upload.sh webkit-ubuntu-18.04 >/tmp/$(basename $0)--webkit.log || true

View file

@ -63,6 +63,4 @@ fi
touch "$IS_FIRST_RUN_FILE"
git pull origin master
../checkout_build_archive_upload.sh webkit-gtk-ubuntu-20.04 >/tmp/$(basename $0)--webkit-gtk.log || true
../checkout_build_archive_upload.sh webkit-wpe-ubuntu-20.04 >/tmp/$(basename $0)--webkit-wpe.log || true
../checkout_build_archive_upload.sh webkit-gtk-wpe-ubuntu-20.04 >/tmp/$(basename $0)--webkit-gtk-wpe.log || true
../checkout_build_archive_upload.sh webkit-ubuntu-20.04 >/tmp/$(basename $0)--webkit.log || true

View file

@ -32,7 +32,6 @@ fi
BROWSER_NAME=""
EXTRA_BUILD_ARGS=""
EXTRA_ARCHIVE_ARGS=""
BUILD_FLAVOR="$1"
BUILD_BLOB_NAME=""
EXPECTED_HOST_OS=""
@ -56,40 +55,12 @@ elif [[ "$BUILD_FLAVOR" == "firefox-win64" ]]; then
EXTRA_BUILD_ARGS="--win64"
EXPECTED_HOST_OS="MINGW"
BUILD_BLOB_NAME="firefox-win64.zip"
elif [[ "$BUILD_FLAVOR" == "webkit-gtk-ubuntu-18.04" ]]; then
BROWSER_NAME="webkit"
EXTRA_BUILD_ARGS="--gtk"
EXTRA_ARCHIVE_ARGS="--gtk"
EXPECTED_HOST_OS="Ubuntu"
EXPECTED_HOST_OS_VERSION="18.04"
BUILD_BLOB_NAME="minibrowser-gtk-ubuntu-18.04.zip"
elif [[ "$BUILD_FLAVOR" == "webkit-wpe-ubuntu-18.04" ]]; then
BROWSER_NAME="webkit"
EXTRA_BUILD_ARGS="--wpe"
EXTRA_ARCHIVE_ARGS="--wpe"
EXPECTED_HOST_OS="Ubuntu"
EXPECTED_HOST_OS_VERSION="18.04"
BUILD_BLOB_NAME="minibrowser-wpe-ubuntu-18.04.zip"
elif [[ "$BUILD_FLAVOR" == "webkit-gtk-wpe-ubuntu-18.04" ]]; then
elif [[ "$BUILD_FLAVOR" == "webkit-ubuntu-18.04" ]]; then
BROWSER_NAME="webkit"
EXPECTED_HOST_OS="Ubuntu"
EXPECTED_HOST_OS_VERSION="18.04"
BUILD_BLOB_NAME="minibrowser-gtk-wpe-ubuntu-18.04.zip"
elif [[ "$BUILD_FLAVOR" == "webkit-gtk-ubuntu-20.04" ]]; then
BROWSER_NAME="webkit"
EXTRA_BUILD_ARGS="--gtk"
EXTRA_ARCHIVE_ARGS="--gtk"
EXPECTED_HOST_OS="Ubuntu"
EXPECTED_HOST_OS_VERSION="20.04"
BUILD_BLOB_NAME="minibrowser-gtk-ubuntu-20.04.zip"
elif [[ "$BUILD_FLAVOR" == "webkit-wpe-ubuntu-20.04" ]]; then
BROWSER_NAME="webkit"
EXTRA_BUILD_ARGS="--wpe"
EXTRA_ARCHIVE_ARGS="--wpe"
EXPECTED_HOST_OS="Ubuntu"
EXPECTED_HOST_OS_VERSION="20.04"
BUILD_BLOB_NAME="minibrowser-wpe-ubuntu-20.04.zip"
elif [[ "$BUILD_FLAVOR" == "webkit-gtk-wpe-ubuntu-20.04" ]]; then
elif [[ "$BUILD_FLAVOR" == "webkit-ubuntu-20.04" ]]; then
BROWSER_NAME="webkit"
EXPECTED_HOST_OS="Ubuntu"
EXPECTED_HOST_OS_VERSION="20.04"
@ -161,20 +132,6 @@ else
fi
function generate_and_upload_browser_build {
# webkit-gtk-wpe is a special build doesn't need to be built.
if [[ "$BUILD_FLAVOR" == webkit-gtk-wpe-* ]]; then
echo "-- combining binaries together"
if ! ./webkit/download_gtk_and_wpe_and_zip_together.sh $ZIP_PATH; then
return 10
fi
echo "-- uploading"
if ! ./upload.sh $BUILD_BLOB_PATH $ZIP_PATH; then
return 11
fi
return 0
fi
# Other browser flavors follow typical build flow.
echo "-- preparing checkout"
if ! ./prepare_checkout.sh $BROWSER_NAME; then
return 20
@ -191,7 +148,7 @@ function generate_and_upload_browser_build {
fi
echo "-- archiving to $ZIP_PATH"
if ! ./$BROWSER_NAME/archive.sh $ZIP_PATH "$EXTRA_ARCHIVE_ARGS"; then
if ! ./$BROWSER_NAME/archive.sh $ZIP_PATH; then
return 23
fi

View file

@ -1,70 +0,0 @@
#!/bin/bash
set -e
set +x
trap "cd $(pwd -P)" EXIT
cd "$(dirname "$0")"
if [[ ($1 == '--help') || ($1 == '-h') ]]; then
echo "usage: $(basename $0) [webkit-gtk|webkit-wpe] [zip-path]"
echo
echo "Download .zip of a browser build."
echo
echo "NOTE: \$AZ_ACCOUNT_KEY (azure account name) and \$AZ_ACCOUNT_NAME (azure account name)"
echo "env variables are required to download builds from CDN."
exit 0
fi
if [[ (-z $AZ_ACCOUNT_KEY) || (-z $AZ_ACCOUNT_NAME) ]]; then
echo "ERROR: Either \$AZ_ACCOUNT_KEY or \$AZ_ACCOUNT_NAME environment variable is missing."
echo " 'Azure Account Name' and 'Azure Account Key' secrets that are required"
echo " to download builds from Azure CDN."
exit 1
fi
if [[ $# < 1 ]]; then
echo "missing build flavor"
echo "try '$(basename $0) --help' for more information"
exit 1
fi
BUILD_FLAVOR="$1"
BROWSER_NAME=""
BLOB_NAME=""
if [[ "$BUILD_FLAVOR" == "webkit-gtk" ]]; then
BROWSER_NAME="webkit"
BLOB_NAME="minibrowser-gtk.zip"
elif [[ "$BUILD_FLAVOR" == "webkit-wpe" ]]; then
BROWSER_NAME="webkit"
BLOB_NAME="minibrowser-wpe.zip"
else
echo ERROR: unsupported build flavor - "$BUILD_FLAVOR"
exit 1
fi
BUILD_NUMBER=$(head -1 ./$BROWSER_NAME/BUILD_NUMBER)
BLOB_PATH="$BROWSER_NAME/$BUILD_NUMBER/$BLOB_NAME"
if [[ $# < 2 ]]; then
echo "missing path to zip archive to download to"
echo "try '$(basename $0) --help' for more information"
exit 1
fi
ZIP_PATH="$2"
if [[ -f $ZIP_PATH ]]; then
echo "ERROR: $ZIP_PATH exists"
exit 1
fi
if ! [[ $ZIP_PATH == *.zip ]]; then
echo "ERROR: $ZIP_PATH is not a zip archive (must have a .zip extension)"
exit 1
fi
az storage blob download -c builds --account-key $AZ_ACCOUNT_KEY --account-name $AZ_ACCOUNT_NAME -f $ZIP_PATH -n "$BLOB_PATH"
echo "DOWNLOAD SUCCESSFUL!"
echo "-- SRC: $ZIP_PATH"
echo "-- SIZE: $(du -h "$ZIP_PATH" | awk '{print $1}')"
echo "-- DST: $BLOB_PATH"

View file

@ -3,7 +3,7 @@ set -e
set +x
if [[ ("$1" == "-h") || ("$1" == "--help") ]]; then
echo "usage: $(basename $0) [output-absolute-path] [--wpe|--gtk]"
echo "usage: $(basename $0) [output-absolute-path]"
echo
echo "Generate distributable .zip archive from ./checkout folder that was previously built."
echo
@ -11,7 +11,6 @@ if [[ ("$1" == "-h") || ("$1" == "--help") ]]; then
fi
ZIP_PATH=$1
LINUX_FLAVOR=$2
if [[ $ZIP_PATH != /* ]]; then
echo "ERROR: path $ZIP_PATH is not absolute"
exit 1
@ -45,80 +44,101 @@ main() {
fi
}
copyLibrariesForWPE() {
# Expect target directory to be passed in as first argument.
local tmpdir=$1
# copy runner
cp -t $tmpdir $SCRIPTS_DIR/pw_run.sh
# copy all relevant binaries
cp -t $tmpdir ./WebKitBuild/WPE/Release/bin/MiniBrowser ./WebKitBuild/WPE/Release/bin/WPE*Process
# copy all relevant shared objects
# - exclude gstreamer plugins
LD_LIBRARY_PATH="$PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib" ldd WebKitBuild/WPE/Release/bin/MiniBrowser \
| grep -o '[^ ]*WebKitBuild/WPE/[^ ]*' \
| grep -v '/libgst.*so' \
| xargs cp -t $tmpdir
LD_LIBRARY_PATH="$PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib" ldd WebKitBuild/WPE/Release/bin/WPENetworkProcess \
| grep -o '[^ ]*WebKitBuild/WPE/[^ ]*' \
| grep -v '/libgst.*so' \
| xargs cp -t $tmpdir
LD_LIBRARY_PATH="$PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib" ldd WebKitBuild/WPE/Release/bin/WPEWebProcess \
| grep -o '[^ ]*WebKitBuild/WPE/[^ ]*' \
| grep -v '/libgst.*so' \
| xargs cp -t $tmpdir
# Copy some wayland libraries required for Web Process
if ls WebKitBuild/WPE/DependenciesWPE/Root/lib/libva\-* 2>&1 >/dev/null; then
cp -d -t $tmpdir WebKitBuild/WPE/DependenciesWPE/Root/lib/libva\-*
fi
# Injected bundle is loaded dynamicly via dlopen => not bt listed by ldd.
cp -t $tmpdir WebKitBuild/WPE/Release/lib/libWPEInjectedBundle.so
if test -d $PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib/gio/modules/; then
mkdir -p $tmpdir/gio/modules
cp -t $tmpdir/gio/modules $PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib/gio/modules/*
fi
cd $tmpdir
ln -s libWPEBackend-fdo-1.0.so.1 libWPEBackend-fdo-1.0.so
cd -
# Strip copied binaries.
cd $tmpdir
strip --strip-unneeded * || true
cd -
}
copyLibrariesForGTK() {
# Expect target directory to be passed in as first argument.
local tmpdir=$1
# copy runner
cp -t $tmpdir $SCRIPTS_DIR/pw_run.sh
# copy all relevant binaries
cp -t $tmpdir ./WebKitBuild/GTK/Release/bin/MiniBrowser ./WebKitBuild/GTK/Release/bin/WebKit*Process
# copy all relevant shared objects
# - exclude gstreamer plugins
# - exclude libdrm
LD_LIBRARY_PATH="$PWD/WebKitBuild/GTK/DependenciesGTK/Root/lib" ldd WebKitBuild/GTK/Release/bin/MiniBrowser \
| grep -o '[^ ]*WebKitBuild/GTK/[^ ]*' \
| grep -v '/libgst.*so' \
| grep -v '/libdrm.so' \
| xargs cp -t $tmpdir
# Injected bundle is loaded dynamicly via dlopen => not bt listed by ldd.
cp -t $tmpdir WebKitBuild/GTK/Release/lib/libwebkit2gtkinjectedbundle.so
if test -d $PWD/WebKitBuild/GTK/DependenciesGTK/Root/lib/gio/modules; then
mkdir -p $tmpdir/gio/modules
cp -t $tmpdir/gio/modules $PWD/WebKitBuild/GTK/DependenciesGTK/Root/lib/gio/modules/*
fi
# we failed to nicely build libgdk_pixbuf - expect it in the env
rm -f $tmpdir/libgdk_pixbuf*
# tar resulting directory and cleanup TMP.
cd $tmpdir
strip --strip-unneeded * || true
cd -
}
createZipForLinux() {
# create a TMP directory to copy all necessary files
local tmpdir=$(mktemp -d -t webkit-deploy-XXXXXXXXXX)
mkdir -p $tmpdir
mkdir -p $tmpdir/minibrowser-gtk
mkdir -p $tmpdir/minibrowser-wpe
# copy runner
cp -t $tmpdir $SCRIPTS_DIR/pw_run.sh
# copy protocol
node $SCRIPTS_DIR/concat_protocol.js > $tmpdir/protocol.json
if [[ "$LINUX_FLAVOR" == "--wpe" ]]; then
# copy all relevant binaries
cp -t $tmpdir ./WebKitBuild/WPE/Release/bin/MiniBrowser ./WebKitBuild/WPE/Release/bin/WPE*Process
# copy all relevant shared objects
# - exclude gstreamer plugins
LD_LIBRARY_PATH="$PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib" ldd WebKitBuild/WPE/Release/bin/MiniBrowser \
| grep -o '[^ ]*WebKitBuild/WPE/[^ ]*' \
| grep -v '/libgst.*so' \
| xargs cp -t $tmpdir
LD_LIBRARY_PATH="$PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib" ldd WebKitBuild/WPE/Release/bin/WPENetworkProcess \
| grep -o '[^ ]*WebKitBuild/WPE/[^ ]*' \
| grep -v '/libgst.*so' \
| xargs cp -t $tmpdir
LD_LIBRARY_PATH="$PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib" ldd WebKitBuild/WPE/Release/bin/WPEWebProcess \
| grep -o '[^ ]*WebKitBuild/WPE/[^ ]*' \
| grep -v '/libgst.*so' \
| xargs cp -t $tmpdir
# Copy libvpx.so.5 as Ubuntu 20.04 comes with libvpx.so.6
ldd WebKitBuild/WPE/Release/bin/MiniBrowser | grep -o '[^ ]*\/libvpx.so.5[^ ]*' | xargs cp -t $tmpdir
# Copy some wayland libraries required for Web Process t
if ls WebKitBuild/WPE/DependenciesWPE/Root/lib/libva\-* 2>&1 >/dev/null; then
cp -d -t $tmpdir WebKitBuild/WPE/DependenciesWPE/Root/lib/libva\-*
fi
# Injected bundle is loaded dynamicly via dlopen => not bt listed by ldd.
cp -t $tmpdir WebKitBuild/WPE/Release/lib/libWPEInjectedBundle.so
if test -d $PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib/gio/modules/; then
mkdir -p $tmpdir/gio/modules
cp -t $tmpdir/gio/modules $PWD/WebKitBuild/WPE/DependenciesWPE/Root/lib/gio/modules/*
fi
cd $tmpdir
ln -s libWPEBackend-fdo-1.0.so.1 libWPEBackend-fdo-1.0.so
cd -
elif [[ "$LINUX_FLAVOR" == "--gtk" ]]; then
# copy all relevant binaries
cp -t $tmpdir ./WebKitBuild/GTK/Release/bin/MiniBrowser ./WebKitBuild/GTK/Release/bin/WebKit*Process
# copy all relevant shared objects
# - exclude gstreamer plugins
# - exclude libdrm
LD_LIBRARY_PATH="$PWD/WebKitBuild/GTK/DependenciesGTK/Root/lib" ldd WebKitBuild/GTK/Release/bin/MiniBrowser \
| grep -o '[^ ]*WebKitBuild/GTK/[^ ]*' \
| grep -v '/libgst.*so' \
| grep -v '/libdrm.so' \
| xargs cp -t $tmpdir
# Injected bundle is loaded dynamicly via dlopen => not bt listed by ldd.
cp -t $tmpdir WebKitBuild/GTK/Release/lib/libwebkit2gtkinjectedbundle.so
# Copy libvpx.so.5 as Ubuntu 20.04 comes with libvpx.so.6
ldd WebKitBuild/GTK/Release/bin/MiniBrowser | grep -o '[^ ]*\/libvpx.so.5[^ ]*' | xargs cp -t $tmpdir
if test -d $PWD/WebKitBuild/GTK/DependenciesGTK/Root/lib/gio/modules; then
mkdir -p $tmpdir/gio/modules
cp -t $tmpdir/gio/modules $PWD/WebKitBuild/GTK/DependenciesGTK/Root/lib/gio/modules/*
fi
# we failed to nicely build libgdk_pixbuf - expect it in the env
rm -f $tmpdir/libgdk_pixbuf*
else
echo "ERROR: must specify --gtk or --wpe"
exit 1
fi
copyLibrariesForWPE $tmpdir/minibrowser-wpe
copyLibrariesForGTK $tmpdir/minibrowser-gtk
# tar resulting directory and cleanup TMP.
cd $tmpdir
strip --strip-unneeded * || true
zip --symlinks -r $ZIP_PATH ./
cd -
rm -rf $tmpdir

View file

@ -1,79 +0,0 @@
#!/bin/bash
set -e
set +x
if [[ ("$1" == "-h") || ("$1" == "--help") ]]; then
echo "usage: $(basename $0) [ZIP-PATH]"
echo
echo "Generate a single .zip archive that contains both gtk and wpe builds"
echo
exit 0
fi
if [[ "$(uname)" != "Linux" ]]; then
echo "ERROR: this script works only on linux"
echo
exit 1
fi
ZIP_PATH="$1"
if [[ $ZIP_PATH != /* ]]; then
echo "ERROR: path $ZIP_PATH is not absolute"
exit 1
fi
if [[ $ZIP_PATH != *.zip ]]; then
echo "ERROR: path $ZIP_PATH must have .zip extension"
exit 1
fi
if [[ -f $ZIP_PATH ]]; then
echo "ERROR: path $ZIP_PATH exists; can't do anything."
exit 1
fi
if ! [[ -d $(dirname $ZIP_PATH) ]]; then
echo "ERROR: folder for path $($ZIP_PATH) does not exist."
exit 1
fi
trap "cd $(pwd -P)" EXIT
cd "$(dirname "$0")"
# create a TMP directory to copy all necessary files
TMPDIR=$(mktemp -d -t webkit-deploy-XXXXXXXXXX)
GTK_ZIP_PATH=$(mktemp -t -u minibrowser-gtk-XXXXXX.zip)
WPE_ZIP_PATH=$(mktemp -t -u minibrowser-wpe-XXXXXX.zip)
../download.sh webkit-gtk $GTK_ZIP_PATH
../download.sh webkit-wpe $WPE_ZIP_PATH
# Create directory
mkdir -p $TMPDIR
# copy runner
cp -t $TMPDIR ./pw_run.sh
pushd $TMPDIR
# Copy MiniBrowser-GTK
mkdir minibrowser-gtk
pushd minibrowser-gtk
cp $GTK_ZIP_PATH archive.zip
unzip archive.zip
rm archive.zip
popd
# Copy MiniBrowser-WPE
mkdir minibrowser-wpe
pushd minibrowser-wpe
cp $WPE_ZIP_PATH archive.zip
unzip archive.zip
rm archive.zip
popd
mv minibrowser-gtk/protocol.json .
rm minibrowser-wpe/protocol.json
zip --symlinks -r $ZIP_PATH ./
popd
rm -rf $TMPDIR
rm -rf $WPE_ZIP_PATH
rm -rf $GTK_ZIP_PATH