From c1352b66c07603175e823d2768797435d2fdaf9c Mon Sep 17 00:00:00 2001 From: FriwiDev Date: Wed, 29 Dec 2021 19:28:48 +0100 Subject: [PATCH] Delta builds for Linux and MacOSX (build in <2 min with docker) --- DockerfileLinux | 8 +++++-- DockerfileLinuxARMPrebuild | 2 ++ compile_linux.sh | 39 +++++++++++++++++++++++++++++++---- compile_macosx.sh | 5 ++++- scripts/run_linux.sh | 28 +++++++++++++++++++++++-- scripts/run_linux_prebuild.sh | 5 ++++- 6 files changed, 77 insertions(+), 10 deletions(-) diff --git a/DockerfileLinux b/DockerfileLinux index daea07d..61d9091 100644 --- a/DockerfileLinux +++ b/DockerfileLinux @@ -1,5 +1,7 @@ FROM friwidev/jcefdocker:linux-latest AS stage +LABEL jcefbuild=true + #Declare build type argument (Release or Debug) ARG BUILD_TYPE @@ -30,5 +32,7 @@ RUN ./run_linux.sh #Export built files FROM scratch AS export-stage COPY --from=stage /jcef/binary_distrib.tar.gz . - - +COPY --from=stage /jcef/target . +COPY --from=stage /jcef/third_party third_party +COPY --from=stage /jcef/buildtools buildtools +COPY --from=stage /jcef/jcef_build jcef_build diff --git a/DockerfileLinuxARMPrebuild b/DockerfileLinuxARMPrebuild index adf12f6..4231212 100644 --- a/DockerfileLinuxARMPrebuild +++ b/DockerfileLinuxARMPrebuild @@ -3,6 +3,8 @@ FROM friwidev/jcefdocker:linux-latest AS stage +LABEL jcefbuild=true + #Declare build type argument (Release or Debug) ARG BUILD_TYPE diff --git a/compile_linux.sh b/compile_linux.sh index cd9e4ce..3356a23 100755 --- a/compile_linux.sh +++ b/compile_linux.sh @@ -19,20 +19,51 @@ mkdir out mkdir out/linux32 touch out/linux32/prebuilt.txt +#Cache build image to not download it again each time (speedup for local builds) +docker pull friwidev/jcefdocker:linux-latest + #Execute buildx with linux dockerfile and output to current directory if [ $# -eq 2 ] then if [ $1 == "arm/v6" ] then rm -rf out/linux32 - docker buildx build --platform=linux/386 --build-arg TARGETARCH=386 --build-arg BUILD_TYPE=$2 --build-arg REPO=https://bitbucket.org/chromiumembedded/java-cef.git --build-arg REF=master --file DockerfileLinuxARMPrebuild --output out . + docker buildx build --no-cache --progress=plain --platform=linux/386 --build-arg TARGETARCH=386 --build-arg BUILD_TYPE=$2 --build-arg REPO=https://bitbucket.org/chromiumembedded/java-cef.git --build-arg REF=master --file DockerfileLinuxARMPrebuild --output out . fi - docker buildx build --platform=linux/$1 --build-arg TARGETARCH=$1 --build-arg BUILD_TYPE=$2 --build-arg REPO=https://bitbucket.org/chromiumembedded/java-cef.git --build-arg REF=master --file DockerfileLinux --output out . + docker buildx build --no-cache --progress=plain --platform=linux/$1 --build-arg TARGETARCH=$1 --build-arg BUILD_TYPE=$2 --build-arg REPO=https://bitbucket.org/chromiumembedded/java-cef.git --build-arg REF=master --file DockerfileLinux --output out . else if [ $1 == "arm/v6" ] then rm -rf out/linux32 - docker buildx build --platform=linux/386 --build-arg TARGETARCH=386 --build-arg BUILD_TYPE=$2 --build-arg REPO=$3 --build-arg REF=$4 --file DockerfileLinuxARMPrebuild --output out . + docker buildx build --no-cache --progress=plain jcefbuild:latest --platform=linux/386 --build-arg TARGETARCH=386 --build-arg BUILD_TYPE=$2 --build-arg REPO=$3 --build-arg REF=$4 --file DockerfileLinuxARMPrebuild --output out . fi - docker buildx build --platform=linux/$1 --build-arg TARGETARCH=$1 --build-arg BUILD_TYPE=$2 --build-arg REPO=$3 --build-arg REF=$4 --file DockerfileLinux --output out . + docker buildx build --no-cache --progress=plain --platform=linux/$1 --build-arg TARGETARCH=$1 --build-arg BUILD_TYPE=$2 --build-arg REPO=$3 --build-arg REF=$4 --file DockerfileLinux --output out . fi +docker builder prune -f --filter "label=jcefbuild=true" + +#Cleanup output dir +rm -rf out/linux32 +rm -f out/third_party/cef/*.bz2 out/third_party/cef/*.sha1 + +# Check if the cef download was performed. If so, move third_party dir to jcef dir +export downloaded=0 +for f in out/third_party/cef/cef_binary_*; do + test -d "$f" || continue + #We found a matching dir + export downloaded=1 + break +done +if [ "$downloaded" -eq "1" ]; then + rm -rf jcef/third_party + mv out/third_party jcef +fi + +# Check if the clang download was performed. If so, move it to jcef dir +if [ -f "out/buildtools/clang-format" ]; then + rm -rf jcef/tools/buildtools/linux64 + mv out/buildtools jcef/tools/buildtools/linux64 +fi + +#Move jcef_build +rm -rf jcef/jcef_build +mv out/jcef_build jcef/jcef_build diff --git a/compile_macosx.sh b/compile_macosx.sh index 75ba6a8..28d73c6 100755 --- a/compile_macosx.sh +++ b/compile_macosx.sh @@ -42,7 +42,10 @@ fi # Create and enter the `jcef_build` directory. # The `jcef_build` directory name is required by other JCEF tooling # and should not be changed. -mkdir jcef_build && cd jcef_build +if [ ! -d "jcef_build" ]; then + mkdir jcef_build +fi +cd jcef_build # MacOS: Generate amd64/arm64 Makefiles. if [ ${TARGETARCH} == 'amd64' ]; then diff --git a/scripts/run_linux.sh b/scripts/run_linux.sh index 70ec6ca..0225162 100644 --- a/scripts/run_linux.sh +++ b/scripts/run_linux.sh @@ -26,7 +26,7 @@ if [ ! -f "/jcef/README.md" ]; then else echo "Found existing files to build" cd /jcef -fi +fi #CMakeLists patching python3 /builder/patch_cmake.py CMakeLists.txt /builder/CMakeLists.txt.patch @@ -34,7 +34,19 @@ python3 /builder/patch_cmake.py CMakeLists.txt /builder/CMakeLists.txt.patch # Create and enter the `jcef_build` directory. # The `jcef_build` directory name is required by other JCEF tooling # and should not be changed. -mkdir jcef_build && cd jcef_build +if [ ! -d "jcef_build" ]; then + mkdir jcef_build +fi +cd jcef_build + +# Check if the download was already performed. If so, we wont send it outside of the container at the end +export already_downloaded=0 +for f in ../third_party/cef/cef_binary_*; do + test -d "$f" || continue + #We found a matching dir + export already_downloaded=1 + break +done # Linux: Generate 32/64-bit Unix Makefiles. cmake -G "Ninja" -DPROJECT_ARCH=${TARGETARCH} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} .. @@ -77,4 +89,16 @@ else #Replace natives on armv6 if [ ${TARGETARCH} == 'arm/v6' ]; then (rm bin/gluegen-rt-natives* && rm bin/jogl-all-natives* && cp /natives/gluegen-rt-natives-linux-armv6hf.jar bin && cp /natives/jogl-all-natives-linux-armv6hf.jar bin) fi fi + +#Export binaries tar -czvf ../../binary_distrib.tar.gz * +mkdir ../../target +mv * ../../target + +#Do not export third_party if already exported (it is quite large) +if [ "$already_downloaded" -eq "1" ]; then + rm -rf ../../third_party/* +fi + +#Export clang +mv ../../tools/buildtools/linux64 ../../buildtools diff --git a/scripts/run_linux_prebuild.sh b/scripts/run_linux_prebuild.sh index 60b1b00..56ce161 100644 --- a/scripts/run_linux_prebuild.sh +++ b/scripts/run_linux_prebuild.sh @@ -28,7 +28,10 @@ fi # Create and enter the `jcef_build` directory. # The `jcef_build` directory name is required by other JCEF tooling # and should not be changed. -mkdir jcef_build && cd jcef_build +if [ ! -d "jcef_build" ]; then + mkdir jcef_build +fi +cd jcef_build #Compile JCEF java classes cd ../tools