X-Git-Url: https://git.xonotic.org/?p=xonotic%2Fxonotic.git;a=blobdiff_plain;f=misc%2Ftools%2Fall%2Frelease.subr;h=b15141dc860a2949329e101ccde89f60232c8de1;hp=3785d61c7c3b8c077a85f1fb14cd6721a22acbab;hb=HEAD;hpb=5288b1fab1f4d370c3685dfc7d01222c9c9229d2 diff --git a/misc/tools/all/release.subr b/misc/tools/all/release.subr index 3785d61c..54f7c8c5 100644 --- a/misc/tools/all/release.subr +++ b/misc/tools/all/release.subr @@ -2,6 +2,13 @@ release_common() { export LC_ALL=C + if [ -z "$MAKEFLAGS" ]; then + ncpus=`getconf _NPROCESSORS_ONLN 2>/dev/null || getconf NPROCESSORS_ONLN 2>/dev/null || echo 1` + if [ $ncpus -gt 1 ]; then + export MAKEFLAGS=-j$ncpus + fi + fi + release_args="$cmd $*" msg "*** $release_args: start" release_starttime=`date +%s` @@ -29,15 +36,15 @@ release_common() # try to create a hardlink if ln -f "$release_src/.git/HEAD" "$release_dst/.hardlink-test"; then rm -f "$release_dst/.hardlink-test" - { - verbose cd "$release_src" - git ls-files HEAD -- "$@" - } | { + git -C "$release_src" ls-files HEAD -- "$@" | while IFS= read -r F; do case "$F" in */*) mkdir -p "$release_dst/${F%/*}" ;; esac verbose ln -f "$release_src/$F" "$release_dst/$F" done - } + git -C "$release_src" ls-tree -rd --name-only HEAD -- "$@" | + while IFS= read -r D; do + touch -cr "$release_src/$D" "$release_dst/$D" + done else { verbose cd "$release_src" @@ -53,26 +60,25 @@ release_common() { archive=$1; shift - # get rid of this hack once we have done large enough changes - # to be no longer rsync compatible (and then enable the below - # code) + # bones_was_here: 7za still doesn't store symlinks correctly (even with -snl) + # but it does now store the executable bit. + # compared to advzip, 7za achieves very similar final zip size, is multi threaded, + # and avoids https://gitlab.com/xonotic/xonotic/-/issues/371 + sevenzipflags=-mx=9 zipflags=-9 - find "$@" -exec touch -d "2001-01-01 01:01:01 +0000" {} \+ # ugly hack to make the pk3 files rsync-friendly + # git restore-mtime replaces this hack: + #find "$@" -exec touch -d "2001-01-01 01:01:01 +0000" {} \+ # ugly hack to make the pk3 files rsync-friendly ziplist=`mktemp` - find "$@" -xtype f \( -executable -or -type l \) -print | sed 's,\([./][^./]*$\),\1 \1,' | sort -k2 | cut -d\ -f1 > "$ziplist" + find "$@" -xtype f \( -type l \) -print | sed 's,\([./][^./]*$\),\1 \1,' | sort -k2 | cut -d\ -f1 > "$ziplist" 7za a -tzip $sevenzipflags -x@"$ziplist" "$archive" "$@" || true zip $zipflags -y -@<"$ziplist" "$archive" || true rm -f "$ziplist" - - #zipflags=-1r - #find "$@" -exec touch -d "2001-01-01 01:01:01 +0000" {} \+ # ugly hack to make the pk3 files rsync-friendly - #zip $zipflags -y "$archive" "$@" || true - #advzip -4 "$archive" } mkzip() { + # not using this currently, see above archive=$1; shift zipflags=-1ry zip $zipflags "$archive" "$@" || true @@ -131,9 +137,8 @@ case "$cmd" in # host names are reachable and have a compile # infrastructure set up: # - xonotic-build-linux64 (with gcc on x86_64) - # - xonotic-build-win32 (with i586-mingw32msvc-g++) - # - xonotic-build-win64 (with amd64-mingw32msvc-g++ - # and amd64-mingw32msvc-g++) + # - xonotic-build-win32 (with i686-w64-mingw32) + # - xonotic-build-win64 (with x86_64-w64-mingw32) # - xonotic-build-osx (with Xcode and SDL.framework) # - AMD Compressonator installed in WINE # - ResEdit installed in WINE @@ -144,7 +149,7 @@ case "$cmd" in # (YYYYMMDD) exit 1 ;; - release) + release|RC*) msg "Building a FINISHED RELEASE" getversion data/xonotic-data.pk3dir verbose "$SELF" each -k git tag -m "TEMP TAG" -f "xonotic-v$versionstr" @@ -158,7 +163,7 @@ case "$cmd" in if [ -n "$RELEASEDATE" ]; then verbose $ECHO "$RELEASEDATE" > Xonotic/stamp.txt case "$RELEASETYPE" in - release) + release|RC*) verbose $ECHO "${RELEASEDATE%_*}" > Xonotic/pk3stamp.txt ;; *) @@ -169,39 +174,94 @@ case "$cmd" in verbose date +%Y%m%d > Xonotic/stamp.txt verbose date +%Y%m%d > Xonotic/pk3stamp.txt fi - release_git_extract_dir "." "Xonotic" Docs misc server xonotic-linux-glx.sh xonotic-linux-sdl.sh xonotic-linux-dedicated.sh Makefile misc/buildfiles key_0.d0pk COPYING GPL-2 GPL-3 + release_git_extract_dir "." "Xonotic" Docs misc/buildfiles/osx misc/logos misc/tools server xonotic-linux-sdl.sh xonotic-linux-dedicated.sh Makefile key_0.d0pk COPYING GPL-2 GPL-3 ( verbose cd Xonotic verbose mkdir data source source/darkplaces source/gmqcc source/d0_blind_id mapping - verbose rm -rf misc/builddeps - verbose mv misc/buildfiles/win32 bin32 || true - verbose mv bin32/SDL.dll . || true - verbose mv misc/buildfiles/win64 bin64 || true verbose mv misc/buildfiles/osx/* . || true verbose rm -rf misc/buildfiles - verbose rm -rf misc/pki + verbose rm -rf misc/tools/all ) release_git_extract_dir "darkplaces" "Xonotic/source/darkplaces" . release_git_extract_dir "gmqcc" "Xonotic/source/gmqcc" . release_git_extract_dir "data/xonotic-data.pk3dir" "Xonotic/source" qcsrc Makefile release_git_extract_dir "d0_blind_id" "Xonotic/source/d0_blind_id" . - ( - verbose cd Xonotic/source/d0_blind_id - verbose sh autogen.sh - ) +# bones_was_here: the main Makefile runs autogen.sh (and configure) when necessary +# ( +# verbose cd Xonotic/source/d0_blind_id +# verbose sh autogen.sh +# ) rm -f Xonotic/key_15.d0pk ( verbose cd Xonotic/mapping - verbose wget http://www.icculus.org/netradiant/files/netradiant-1.5.0-20120301.tar.bz2 - verbose wget http://www.icculus.org/netradiant/files/netradiant-1.5.0-20120301-win32-7z.exe - for X in *-7z.exe; do - 7za x "$X" - rm -f "$X" - done + # bones_was_here: don't ship 10 year old netradiant + # also seems better to update netradiant independently of xonotic versions + #verbose wget http://www.icculus.org/netradiant/files/netradiant-1.5.0-20120301.tar.bz2 + #verbose wget http://www.icculus.org/netradiant/files/netradiant-1.5.0-20120301-win32-7z.exe + #for X in *-7z.exe; do + # 7za x "$X" + # rm -f "$X" + #done # TODO possibly include other tools? + printf "Please find links to NetRadiant builds at https://netradiant.gitlab.io or https://xonotic.org/download\nNetRadiant source available at https://gitlab.com/xonotic/netradiant\n" > NetRadiant\ links.txt ) ./all each git rev-parse HEAD > Xonotic/misc/git-revisions.txt ;; + release-compile-run) + release_common + host=$1 + buildpath=$2 + maketargets=$3 + makeflags=$4 + srcdir=$5 + depsdir=$6 + targetfiles=$7 + schroot= + set -x + if [ -z "$targetfiles" ]; then + exit + fi + case " $HOSTS_THAT_ARE_DISABLED " in + *\ $host\ *) + exit + ;; + esac + case " $HOSTS_THAT_ARE_SCHROOT " in + *\ $host\ *) + schroot="schroot -c $host --preserve-environment --" + ;; + esac + case " $HOSTS_THAT_ARE_MYSELF $HOSTS_THAT_ARE_SCHROOT " in + *\ $host\ *) + verbose rsync --delete -zLvaSHP "$srcdir"/ "$buildpath/" + verbose rsync --delete -zLvaSHP misc/logos/icons_ico/xonotic.ico "$buildpath"/darkplaces.ico + verbose rsync --delete -zLvaSHP "$depsdir"/*/ "$buildpath.deps/" + verbose ln -snf "$buildpath.deps" "$buildpath/.deps" + verbose eval $schroot make -C "$buildpath" clean $maketargets $makeflags + for f in $targetfiles; do + verbose mv "$buildpath/${f%:*}" "${f##*:}" || true + done + ;; + *) + verbose rsync --delete -zLvaSHP "$srcdir"/ "$host":"$buildpath/" + verbose rsync --delete -zLvaSHP misc/logos/icons_ico/xonotic.ico "$host":"$buildpath"/darkplaces.ico + verbose rsync --delete -zLvaSHP "$depsdir"/*/ "$host":"$buildpath.deps/" + verbose ssh "$host" "[ -f /etc/profile ] && . /etc/profile; [ -f ~/.profile ] && . ~/.profile; export LC_ALL=C; ln -snf $buildpath.deps $buildpath/.deps && cd $buildpath && nice -`nice` make clean $maketargets $makeflags" + for f in $targetfiles; do + verbose rsync -zvaSHP "$host:$buildpath/${f%:*}" "${f##*:}" || true + done + ;; + esac + ;; + release-compile) + release_common + suffix=$1 + makeflags=$2 + darkplaces_maketargets=$3 + darkplaces_files=$4 + host=xonotic-build-$suffix + verbose "$SELF" release-compile-run "$host" /tmp/Darkplaces.build."$suffix" "$darkplaces_maketargets" "$makeflags" "Xonotic/source/darkplaces" "$d0/misc/builddeps/$suffix" "$darkplaces_files" + ;; release-getbinary) release_common binary=$1 @@ -217,34 +277,132 @@ case "$cmd" in ;; esac ;; + release-libs-compile) + release_common + arch=$1 + makeflags=$2 + host=$3 + ( + verbose cd "$d0/d0_blind_id" + verbose ./autogen.sh + verbose ./configure --enable-static --disable-shared \ + --host="$host" \ + --prefix="$d0/misc/builddeps/$arch/d0_blind_id" + verbose make clean + verbose eval make "$makeflags" + verbose make install + ) + ;; + release-libs-linux64) + release_common + # TODO schroot support for libs so we can LTO this + verbose "$SELF" release-libs-compile linux64 \ + 'CC="gcc -m64"' \ + x86_64-pc-linux-gnu + ;; + release-dlls-compile) + release_common + arch=$1 + buildpath=$2 + targetdir=$3 + verbose mkdir -p "$targetdir" + ( + cd "$d0/misc/buildsrc" + verbose ./dlls.sh clean "$buildpath" + verbose ./dlls.sh all "$buildpath" "$arch" + ) + verbose cp -v $buildpath/out/$arch/* "$targetdir" + + # SDL2 + verbose mkdir -p "$d0/misc/builddeps/$arch/" + verbose rsync --delete -avP "$buildpath/pkg/$arch/sdl" "$d0/misc/builddeps/$arch/" + # PKG_CONFIG HACK: avoids needing to modify sdl2-config + verbose printf '#!/bin/sh\nprintf ../../../.deps/lib\n' > "$d0/misc/builddeps/$arch/sdl/libdir_hack.sh" + verbose chmod +x "$d0/misc/builddeps/$arch/sdl/libdir_hack.sh" + ;; + release-dlls-win32) + release_common + verbose "$SELF" release-dlls-compile win32 /tmp/dlls Xonotic/bin32 + ;; + release-dlls-win64) + release_common + verbose "$SELF" release-dlls-compile win64 /tmp/dlls Xonotic/bin64 + ;; + release-libs-osx) + release_common + verbose mkdir -p "$d0/misc/builddeps/osx" + ( + verbose cd "$d0/misc/builddeps/osx" + verbose rm -rf sdl SDL2 + verbose wget https://libsdl.org/release/SDL2-2.28.5.dmg + verbose 7z x *.dmg + # 7z still fails to extract working symlinks from any archive format, hackaround: + verbose ln -sf $(cat SDL2/SDL2.framework/Versions/Current) SDL2/SDL2.framework/Versions/Current + verbose ln -sf $(cat SDL2/SDL2.framework/Headers) SDL2/SDL2.framework/Headers + verbose ln -sf $(cat SDL2/SDL2.framework/Resources) SDL2/SDL2.framework/Resources + verbose ln -sf $(cat SDL2/SDL2.framework/SDL2) SDL2/SDL2.framework/SDL2 + # THE HEADERS ARE NOT IN A SUBDIRECTORY CALLED SDL2 + # YET SDL2/ WAS ADDED TO ALL THE INCLUDES, WHY?!? + # OLD VERSIONS WITHOUT IT WORKED + verbose sed -i "s/#include xonotic-common.cfg.new + # Following line was included in sed above, ref https://gitlab.com/xonotic/xonotic-data.pk3dir/-/merge_requests/1034 + # s/^gameversion_min [0-9]*/gameversion_min 0/; else # >= 1.0 verbose sed " - s/^set g_xonoticversion [^ ]* /set g_xonoticversion $versionstr /; - s/^gameversion_min [0-9]*/gameversion_min $(( ($gv / 100) * 100 - 100 ))/; + s/^set g_xonoticversion [^ ]* /set g_xonoticversion \"$versionstr\" /; s/^gameversion_max [0-9]*/gameversion_max $(( ($gv / 100) * 100 + 199 ))/; " < xonotic-common.cfg > xonotic-common.cfg.new + # Following line was included in sed above, ref https://gitlab.com/xonotic/xonotic-data.pk3dir/-/merge_requests/1034 + # s/^gameversion_min [0-9]*/gameversion_min $(( ($gv / 100) * 100 - 100 ))/; fi mv xonotic-common.cfg.new xonotic-common.cfg case "$RELEASETYPE" in - release) + release|RC*) echo "" >> xonotic-common.cfg echo "// nicer menu" >> xonotic-common.cfg echo "set menu_watermark \"\"" >> xonotic-common.cfg @@ -481,10 +601,11 @@ case "$cmd" in release_common verbose "$SELF" release-buildpk3s data/font-unifont.pk3dir raw '' verbose "$SELF" release-buildpk3s data/font-xolonium.pk3dir raw '' - verbose "$SELF" release-buildpk3s data/xonotic-data.pk3dir low '-low' normaldds '' normal '-high' - verbose "$SELF" release-buildpk3s data/xonotic-maps.pk3dir low '-low' normaldds '' mapping '-mapping' normal '-high' - verbose "$SELF" release-buildpk3s data/xonotic-music.pk3dir raw '' low '-low' - verbose "$SELF" release-buildpk3s data/xonotic-nexcompat.pk3dir low '-low' normaldds '' normal '-high' + verbose "$SELF" release-buildpk3s data/xonotic-data.pk3dir normaldds '' normal '-high' + verbose "$SELF" release-buildpk3s data/xonotic-maps.pk3dir normaldds '' mapping '-mapping' normal '-high' + verbose "$SELF" release-buildpk3s data/xonotic-music.pk3dir raw '' + verbose "$SELF" release-buildpk3s data/xonotic-nexcompat.pk3dir normaldds '' normal '-high' + verbose "$SELF" release-buildpk3s data/xonotic-xoncompat.pk3dir normaldds '' normal '-high' ;; release-pack-needsx11) release_common @@ -501,64 +622,58 @@ case "$cmd" in release_common stamp=`cat Xonotic/stamp.txt` pk3stamp=`cat Xonotic/pk3stamp.txt` - # exe and dll files do not need +x, so this makes them eligible for 7zip compression too - chmod a-x Xonotic/*.exe Xonotic/*.dll || true - for suffix in '' '-noaes'; do - # build the archives - verbose mkzip Xonotic-$stamp-enginesource$suffix.zip \ - Xonotic/Makefile \ - Xonotic/source/darkplaces/ \ - Xonotic/COPYING Xonotic/GPL-2 Xonotic/GPL-3 - verbose cp Xonotic-$stamp-enginesource$suffix.zip Xonotic-$stamp-engine$suffix.zip - verbose mkzip Xonotic-$stamp-engine$suffix.zip \ - Xonotic/*.dll \ - Xonotic/bin32/*.dll \ - Xonotic/bin64/*.dll \ - Xonotic/*.app \ - Xonotic/xonotic-* \ - Xonotic/xonotic.exe - verbose cp Xonotic-$stamp-engine$suffix.zip Xonotic-$stamp-common$suffix.zip - verbose mkzip Xonotic-$stamp-common$suffix.zip \ - Xonotic/source/d0_blind_id/ \ - Xonotic/source/gmqcc/ \ - Xonotic/source/qcsrc/ \ - Xonotic/Docs \ - Xonotic/misc \ - Xonotic/gmqcc \ - Xonotic/server \ - Xonotic/key_0.d0pk \ - Xonotic/data/font-unifont-$pk3stamp.pk3 \ - Xonotic/data/font-xolonium-$pk3stamp.pk3 - verbose cp Xonotic-$stamp-enginesource$suffix.zip Xonotic-$stamp-source$suffix.zip - verbose mkzip Xonotic-$stamp-source$suffix.zip \ - Xonotic/source/d0_blind_id/ \ - Xonotic/source/gmqcc/ \ - Xonotic/source/qcsrc/ \ - Xonotic/misc/logos - verbose cp Xonotic-$stamp-common$suffix.zip Xonotic-$stamp$suffix.zip - verbose mkzip0 Xonotic-$stamp$suffix.zip \ - Xonotic/data/xonotic-$pk3stamp-data.pk3 \ - Xonotic/data/xonotic-$pk3stamp-maps.pk3 \ - Xonotic/data/xonotic-$pk3stamp-music.pk3 \ - Xonotic/data/xonotic-$pk3stamp-nexcompat.pk3 - verbose cp Xonotic-$stamp-common$suffix.zip Xonotic-$stamp-low$suffix.zip - verbose mkzip0 Xonotic-$stamp-low$suffix.zip \ - Xonotic/data/xonotic-$pk3stamp-data-low.pk3 \ - Xonotic/data/xonotic-$pk3stamp-maps-low.pk3 \ - Xonotic/data/xonotic-$pk3stamp-music-low.pk3 \ - Xonotic/data/xonotic-$pk3stamp-nexcompat-low.pk3 - verbose mv Xonotic-$stamp-common$suffix.zip Xonotic-$stamp-high$suffix.zip - verbose mkzip0 Xonotic-$stamp-high$suffix.zip \ - Xonotic/data/xonotic-$pk3stamp-data-high.pk3 \ - Xonotic/data/xonotic-$pk3stamp-maps-high.pk3 \ - Xonotic/data/xonotic-$pk3stamp-music.pk3 \ - Xonotic/data/xonotic-$pk3stamp-nexcompat-high.pk3 - done - verbose mkzip Xonotic-$stamp-mappingsupport.zip \ + # exe and dll files do not need +x + chmod a-x Xonotic/*.exe Xonotic/*.dll Xonotic/bin64/*.dll Xonotic/bin32/*.dll || true + # build the archives + verbose mkzipr Xonotic-$stamp-enginesource.zip \ + Xonotic/Makefile \ + Xonotic/source/darkplaces/ \ + Xonotic/COPYING Xonotic/GPL-2 Xonotic/GPL-3 + verbose cp Xonotic-$stamp-enginesource.zip Xonotic-$stamp-engine.zip + verbose mkzipr Xonotic-$stamp-engine.zip \ + Xonotic/*.dll \ + Xonotic/bin32/*.dll \ + Xonotic/bin64/*.dll \ + Xonotic/*.app \ + Xonotic/xonotic-* \ + Xonotic/xonotic.exe + verbose cp Xonotic-$stamp-engine.zip Xonotic-$stamp-common.zip + verbose mkzipr Xonotic-$stamp-common.zip \ + Xonotic/source/d0_blind_id/ \ + Xonotic/source/gmqcc/ \ + Xonotic/source/qcsrc/ \ + Xonotic/Docs \ + Xonotic/misc \ + Xonotic/gmqcc \ + Xonotic/server \ + Xonotic/key_0.d0pk \ + Xonotic/data/font-unifont-$pk3stamp.pk3 \ + Xonotic/data/font-xolonium-$pk3stamp.pk3 + verbose cp Xonotic-$stamp-enginesource.zip Xonotic-$stamp-source.zip + verbose mkzipr Xonotic-$stamp-source.zip \ + Xonotic/source/d0_blind_id/ \ + Xonotic/source/gmqcc/ \ + Xonotic/source/qcsrc/ \ + Xonotic/misc/logos + verbose cp Xonotic-$stamp-common.zip Xonotic-$stamp.zip + verbose mkzip0 Xonotic-$stamp.zip \ + Xonotic/data/xonotic-$pk3stamp-data.pk3 \ + Xonotic/data/xonotic-$pk3stamp-maps.pk3 \ + Xonotic/data/xonotic-$pk3stamp-music.pk3 \ + Xonotic/data/xonotic-$pk3stamp-nexcompat.pk3 \ + Xonotic/data/xonotic-$pk3stamp-xoncompat.pk3 + verbose mv Xonotic-$stamp-common.zip Xonotic-$stamp-high.zip + verbose mkzip0 Xonotic-$stamp-high.zip \ + Xonotic/data/xonotic-$pk3stamp-data-high.pk3 \ + Xonotic/data/xonotic-$pk3stamp-maps-high.pk3 \ + Xonotic/data/xonotic-$pk3stamp-music.pk3 \ + Xonotic/data/xonotic-$pk3stamp-nexcompat-high.pk3 \ + Xonotic/data/xonotic-$pk3stamp-xoncompat-high.pk3 + verbose mkzipr Xonotic-$stamp-mappingsupport.zip \ Xonotic/mapping verbose mkzip0 Xonotic-$stamp-mappingsupport.zip \ Xonotic/data/xonotic-$pk3stamp-maps-mapping.pk3 - verbose mkzip gmqcc-$stamp.zip \ + verbose mkzipr gmqcc-$stamp.zip \ Xonotic/gmqcc \ Xonotic/source/gmqcc ;; @@ -566,9 +681,10 @@ case "$cmd" in release_common verbose "$SELF" release-prepare verbose "$SELF" release-maps + verbose "$SELF" release-libs verbose "$SELF" release-engine verbose "$SELF" release-qc - verbose "$SELF" release-pack-needsx11 + verbose "$SELF" release-pack verbose "$SELF" release-zip ;; *)