misc/tools/xonotic-map-compiler-autobuild

   1 #!/bin/sh
   2
   3 set -e
   4
   5 bspdir="$PWD/data"
   6 url_http=https://beta.xonotic.org/autobuild-bsp/
   7 url_ssh=xonotic-beta:./
   8 build_cachedir="$HOME/xonotic-map-compiler.cache/"
   9 screenshot_cachedir="$HOME/xonotic-map-screenshot.cache/"
  10 build_override="-bsp_timeout 3600 -vis_timeout 10800 -light_timeout 18000 -minimap_timeout 900 -scale_timeout 900 -timeout_stealing 0.8"
  11 screenshot_override="9 900 5 +g_ctf 1"
  12
  13 getthemap_fail=false
  14
  15 allmaps()
  16 {
  17         commithash=$1
  18         shift
  19         for F in `git show "${commithash}:maps/"`; do
  20                 case "$F" in
  21                         *.map.options)
  22                                 ;;
  23                         *)
  24                                 continue
  25                                 ;;
  26                 esac
  27                 M=${F%.map.options}
  28                 blobhash=`git rev-parse --revs-only "${commithash}:maps/$M.map.options" || true`-`git rev-parse --revs-only "${commithash}:maps/$M.map" || true`
  29                 case "$blobhash" in
  30                         -*)
  31                                 ;;
  32                         *-)
  33                                 ;;
  34                         *)
  35                                 "$@" "$M" "$blobhash" "$commithash"
  36                                 ;;
  37                 esac
  38         done
  39 }
  40
  41 pre2spam()
  42 {
  43         map=$1
  44         url=$2
  45         branch=$3
  46         hash=$4
  47         lasttime_formatted=$5
  48
  49         branch=${branch##refs/heads/}
  50         branch=${branch##refs/remotes/}
  51         branch=${branch##origin/}
  52         hash=`echo "$hash" | cut -c 1-7`
  53
  54         echo -n "[<font color=\"#FF00FF\">$branch</font> $hash] starting map compile of $map"
  55         if [ -n "$lasttime_formatted" ]; then
  56                 echo ", last time it took $lasttime_formatted"
  57         else
  58                 echo
  59         fi
  60 }
  61
  62 ss2spam()
  63 {
  64         map=$1
  65         url=$2
  66         branch=$3
  67         hash=$4
  68         status=$5
  69
  70         branch=${branch##refs/heads/}
  71         branch=${branch##refs/remotes/}
  72         branch=${branch##origin/}
  73         hash=`echo "$hash" | cut -c 1-7`
  74
  75         s_error=
  76
  77         if [ "$status" -ne 0 ]; then
  78                 s_error="exited with status $status"
  79         fi
  80
  81         if [ -n "$s_error" ]; then
  82                 echo "[<font color=\"#FF00FF\">$branch</font> $hash] screenshots of $map <font color=\"#FF0000\">could not be taken: $s_error</font>"
  83         else
  84                 echo "[<font color=\"#FF00FF\">$branch</font> $hash] screenshots of $map <a href=\"$url\">are available</a>"
  85         fi
  86 }
  87
  88 log2spam()
  89 {
  90         map=$1
  91         url=$2
  92         branch=$3
  93         hash=$4
  94         status=$5
  95         time=$6
  96
  97         hash=`echo "$hash" | cut -c 1-7`
  98
  99         branch=${branch##refs/heads/}
 100         branch=${branch##refs/remotes/}
 101         branch=${branch##origin/}
 102
 103         s_samplesize=
 104         s_failshaders=
 105         s_leaked=
 106         s_error=
 107
 108         if [ "$status" -ne 0 ]; then
 109                 s_error="exited with status $status"
 110         fi
 111         while IFS= read -r L; do
 112                 case "$L" in
 113                         WARNING:\ surface\ at\ *\ too\ large\ for\ desired\ samplesize*)
 114                                 s=${L##* }
 115                                 if [ -z "$s_samplesize" ] || [ "$s" -gt "$s_samplesize" ]; then
 116                                         s_samplesize=$s
 117                                 fi
 118                                 ;;
 119                         WARNING:\ Couldn\'t\ find\ image\ for\ shader\ textures/NULL)
 120                                 # radiant stupid
 121                                 ;;
 122                         WARNING:\ Couldn\'t\ find\ image\ for\ shader\ *)
 123                                 s_failshaders="$s_failshaders ${L##* }"
 124                                 ;;
 125                         ERROR:\ *)
 126                                 if [ -z "$s_error" ]; then
 127                                         s_error=$L
 128                                 fi
 129                                 ;;
 130                         '******* leaked *******')
 131                                 s_leaked=1
 132                                 ;;
 133                         '************ ERROR ************')
 134                                 IFS= read -r s_error
 135                                 ;;
 136                 esac
 137         done
 138         s_failshaders=`echo "$s_failshaders" | sed "s, textures/, ,g"`
 139         s_failshaders=${s_failshaders# }
 140         if [ -n "$s_error" ]; then
 141                 echo -n "[<font color=\"#FF00FF\">$branch</font> $hash] <font color=\"#FF0000\">failed</font>"
 142         else
 143                 echo -n "[<font color=\"#FF00FF\">$branch</font> $hash] <font color=\"#00FF00\">finished</font>"
 144         fi
 145         printf " map compile of <a href=\"$url\">$map</a> in %d:%02d:%02d" $((time/3600)) $((time%3600/60)) $((time%60))
 146         if [ -n "$s_samplesize" ]; then
 147                 echo -n "<br><font color=\"#FF0000\">FIX samplesize >= $s_samplesize</font>"
 148         fi
 149         if [ -n "$s_failshaders" ]; then
 150                 if [ -n "`echo "$s_failshaders" | cut -d ' ' -f 4-`" ]; then
 151                         s_failshaders="`echo "$s_failshaders" | cut -d ' ' -f 1-3`..."
 152                 fi
 153                 echo -n "<br><font color=\"#FF0000\">FIX shaders $s_failshaders</font>"
 154         fi
 155         if [ -n "$s_leaked" ]; then
 156                 echo -n "<br><font color=\"#FF0000\">FIX LEAK</font>"
 157         fi
 158         if [ -n "$s_error" ]; then
 159                 s_error=`echo "$s_error" | sed "s,$PWD/\?,,g"`
 160                 echo -n "<br><font color=\"#FF0000\">ERROR: $s_error</font>"
 161         fi
 162         echo
 163 }
 164
 165 buildthemap()
 166 {
 167         REFNAME=$1
 168         url=$2
 169         M=$3
 170         blobhash=$4
 171         commithash=$5
 172         if lwp-request -m HEAD "$url$M-$blobhash.pk3"; then
 173                 # already compiled
 174                 return
 175         fi
 176
 177         git reset --hard
 178         git clean -xfd
 179         git checkout -f "$commithash"
 180
 181         mkdir -p ~/.xonotic-map-compiler-autobuild/ # to store map compile times in
 182         lasttime=`cat ~/.xonotic-map-compiler-autobuild/"${M##*/}".buildtime 2>/dev/null || true`
 183         if [ -n "$lasttime" ] ; then
 184                 lasttime_formatted=$(printf '%d:%02d:%02d' $((lasttime/3600)) $((lasttime%3600/60)) $((lasttime%60)) )
 185                 echo "Previous build took $lasttime_formatted"
 186         fi
 187         if [ -n "$IRCSPAM" ]; then
 188                 pre2spam "$M" "$url$M-$blobhash.pk3" "$REFNAME" "$commithash" "$lasttime_formatted" | $IRCSPAM || true
 189         fi
 190
 191         t0=`date +%s`
 192
 193         # It's sad that we must enforce this rule by code.
 194         if grep -- '-nosRGB' "maps/$M.map.options" >/dev/null; then
 195                 is_nosrgb=true
 196         else
 197                 is_nosrgb=false
 198         fi
 199         allow_build=true
 200         if $is_nosrgb; then
 201                 case "$REFNAME" in
 202                         master|refs/heads/master|origin/master|refs/remotes/origin/master)
 203                                 if [ -n "$IRCSPAM" ]; then
 204                                         echo "\ 34ERROR: map $M uses -nosRGB. This is not acceptable in master. Please fix." | $IRCSPAM || true
 205                                 fi
 206                                 allow_build=false
 207                                 ;;
 208                         *)
 209                                 if [ -n "$IRCSPAM" ]; then
 210                                         echo "\ 34WARNING:\ f map $M uses -nosRGB. This is not acceptable in master. Please fix." | $IRCSPAM || true
 211                                 fi
 212                                 ;;
 213                 esac
 214         fi
 215
 216         set +e
 217         $allow_build && (
 218                 cd ../..
 219                 misc/tools/xonotic-map-compiler-optionsfile "data/xonotic-maps.pk3dir/maps/$M" $build_override > "data/xonotic-maps.pk3dir/maps/$M.log" 2>&1
 220         )
 221         status=$?
 222         set -e
 223         if ! [ -f "maps/$M.bsp" ]; then
 224                 echo >>"maps/$M.log" "ERROR: No BSP file"
 225         fi
 226         t1=`date +%s`
 227         dt=$(($t1 - $t0))
 228         if [ -n "$IRCSPAM" ]; then
 229                 cat "maps/$M.log" | log2spam "$M" "$url$M-$blobhash.pk3" "$REFNAME" "$commithash" "$status" "$dt" > "maps/$M.irc"
 230         fi
 231         if [ $status -eq 0 ]; then
 232                 echo "$dt" > ~/.xonotic-map-compiler-autobuild/"${M##*/}".buildtime # save time in ~/.xonotic-map-compiler-autobuild/mapname
 233         fi
 234         zip -9r "$M-$blobhash.pk3" "maps/$M.bsp" "maps/$M.log" "maps/$M.irc" "maps/$M/" "maps/$M.lin" "gfx/${M}_mini.tga"
 235         ln -snf "../$M-$blobhash.pk3" "$M.pk3" # from ALL branches, so beware!
 236         cp "$M-$blobhash.pk3" "$M-full-$blobhash.pk3"
 237         zip -9r "$M-full-$blobhash.pk3" `git diff --name-only --diff-filter=ACMRTUXB master...HEAD` || true
 238         ln -snf "../$M-full-$blobhash.pk3" "$M-full.pk3" # from ALL branches, so beware!
 239         rsync -vaSHP "$M-$blobhash.pk3" "$M-full-$blobhash.pk3" "$url_ssh"
 240         rsync -vaSHP "$M.pk3" "$M-full.pk3" "$url_ssh""latest/"
 241         if [ -n "$IRCSPAM" ]; then
 242                 $IRCSPAM < "maps/$M.irc" || true
 243         fi
 244         if [ -n "$BUILD_ONE" ]; then
 245                 exit
 246         fi
 247 }
 248
 249 screenshotthemap()
 250 {
 251         REFNAME=$1
 252         url=$2
 253         M=$3
 254         blobhash=$4
 255         commithash=$5
 256         if ! [ -f "$bspdir/$M-$blobhash.pk3" ]; then
 257                 # not downloaded yet
 258                 return
 259         fi
 260
 261         if lwp-request -m HEAD "$url$M-$blobhash/$M-000000.jpg"; then
 262                 # already done
 263                 return
 264         fi
 265
 266         if ! unzip -l "$bspdir/$M-$blobhash.pk3" "maps/$M.bsp"; then
 267                 # no BSP file
 268                 return
 269         fi
 270
 271         rm -rf ~/.xonotic/data
 272         mkdir -p ~/.xonotic/data
 273         (
 274                 cd ../..
 275                 if [ -n "$DISPLAY" ]; then
 276                         misc/tools/xonotic-map-screenshot "$M" $screenshot_override $DRIVERFLAGS +"scr_screenshot_name \"$M-\""
 277                 else
 278                         case "$DRIVER" in
 279                                 gl)
 280                                         startx "$PWD/misc/tools/xonotic-map-screenshot" "$M" $screenshot_override $DRIVERFLAGS +"scr_screenshot_name \"$M-\"" -- :8
 281                                         ;;
 282                                 soft|*)
 283                                         startx "$PWD/misc/tools/xonotic-map-screenshot" "$M" $screenshot_override +"vid_soft 1" $DRIVERFLAGS +"scr_screenshot_name \"$M-\"" -- /usr/bin/Xvfb :8 -screen 0 1024x768x24
 284                                         ;;
 285                         esac
 286                 fi
 287         ) 2>&1 | tee ~/.xonotic/data/autoscreenshot.log
 288         good=false
 289         if mv ~/.xonotic/data/screenshots "$M-$blobhash"; then
 290                 ss2spam "$M" "$url$M-$blobhash/" "$REFNAME" "$commithash" 0 > ~/.xonotic/data/"$M.ircss"
 291         else
 292                 mkdir "$M-$blobhash"
 293                 ss2spam "$M" "$url$M-$blobhash/" "$REFNAME" "$commithash" 1 > ~/.xonotic/data/"$M.ircss"
 294         fi
 295         mv ~/.xonotic/data/"$M.ircss" "$M-$blobhash"/
 296         mv ~/.xonotic/data/autoscreenshot.log "$M-$blobhash"/
 297         chmod 1777 "$M-$blobhash"
 298         ln -snf "../$M-$blobhash" "$M" # from ALL branches, so beware!
 299         rsync -vaSHP "$M-$blobhash" "$url_ssh"
 300         rsync -vaSHP "$M" "$url_ssh""latest/"
 301         if [ -n "$IRCSPAM" ]; then
 302                 # It takes about 5 minutes for the server to update.
 303                 sleep 360  # Cut it one more minute of slack.
 304
 305                 $IRCSPAM < "$M-$blobhash/$M.ircss" || true
 306         fi
 307         if [ -n "$BUILD_ONE" ]; then
 308                 exit
 309         fi
 310 }
 311
 312 getthemap()
 313 {
 314         url=$1
 315         bspdir_old=$2
 316         bspdir=$3
 317         M=$4
 318         blobhash=$5
 319         commithash=$6
 320         if mv "$bspdir_old/$M-$blobhash.pk3" "$bspdir/$M-$blobhash.pk3"; then
 321                 if unzip -l "$bspdir/$M-$blobhash.pk3" >/dev/null 2>&1; then
 322                         return 0
 323                 fi
 324         fi
 325         if ! wget -c -O "$bspdir/$M-$blobhash.pk3" "$url$M-$blobhash.pk3"; then
 326                 if ! curl -Lo "$bspdir/$M-$blobhash.pk3" "$url$M-$blobhash.pk3"; then
 327                         rm -f "$bspdir/$M-$blobhash.pk3"
 328                         echo "WARNING: could not download $url$M-$blobhash.pk3, maybe not ready yet"
 329                         getthemap_fail=true
 330                         return 0
 331                 fi
 332         fi
 333         if ! unzip -l "$bspdir/$M-$blobhash.pk3"; then
 334                 rm -f "$bspdir/$M-$blobhash.pk3"
 335                 echo "WARNING: could not download $url$M-$blobhash.pk3, invalid zip file"
 336                 getthemap_fail=true
 337                 return 0
 338         fi
 339 }
 340
 341 indexthemap()
 342 {
 343         REFNAME=$1
 344         M=$2
 345         blobhash=$3
 346         commithash=$4
 347         echo "$M $blobhash $commithash $REFNAME"
 348 }
 349
 350 rundownload()
 351 {
 352         mkdir -p "$bspdir" "$bspdir.old"
 353         for b in "$bspdir"/*-????????????????????????????????????????-????????????????????????????????????????.pk3; do
 354                 if [ -e "$b" ]; then
 355                         mv "$b" "$bspdir.old"/
 356                 fi
 357         done
 358         allmaps "HEAD" getthemap "$url_http" "$bspdir.old" "$bspdir"
 359 }
 360
 361 branches()
 362 {
 363         git for-each-ref 'refs/remotes' | grep -vE '    refs/remotes/([^/]*/HEAD|.*/archived/.*)$'
 364 }
 365
 366 runmakeindex()
 367 {
 368         cd data/xonotic-maps.pk3dir
 369         branches | while read -r HASH TYPE REFNAME; do
 370                 allmaps "$HASH" indexthemap "$REFNAME"
 371         done
 372         cd ../..
 373 }
 374
 375 case "$1" in
 376         build)
 377                 cd data/xonotic-maps.pk3dir
 378                 branches | while read -r HASH TYPE REFNAME; do
 379                         if [ -f "$build_cachedir/$HASH" ]; then
 380                                 continue
 381                         fi
 382                         allmaps "$HASH" buildthemap "$REFNAME" "$url_http"
 383                         touch "$build_cachedir/$HASH"
 384                 done
 385                 git checkout -f master
 386                 ;;
 387         screenshot)
 388                 runmakeindex > "branches.idx.new"
 389                 rsync -vaSHP "branches.idx.new" "$url_ssh""branches.idx.new"
 390                 rm -f "branches.idx.new"
 391                 cd data/xonotic-maps.pk3dir
 392                 branches | while read -r HASH TYPE REFNAME; do
 393                         if [ -f "$screenshot_cachedir/$HASH" ]; then
 394                                 continue
 395                         fi
 396
 397                         git reset --hard
 398                         git clean -xfd
 399                         git checkout -f "$HASH"
 400
 401                         getthemap_fail=false
 402                         rundownload
 403
 404                         allmaps "$HASH" screenshotthemap "$REFNAME" "$url_http"
 405
 406                         if $getthemap_fail; then
 407                                 # If any map fetch failed, we've skipped them and thus need to try again later.
 408                                 # Note that we're not going to re-screenshot maps because of this as we'll check for uploaded screenshots first.
 409                                 continue
 410                         fi
 411
 412                         touch "$screenshot_cachedir/$HASH"
 413                 done
 414                 git checkout -f master
 415                 ;;
 416         makeindex)
 417                 runmakeindex
 418                 ;;
 419         download)
 420                 if ! which unzip >/dev/null 2>&1; then
 421                         echo "Cannot find unzip, bailing out"
 422                         exit 1
 423                 fi
 424                 if ! cd data/xonotic-maps.pk3dir >/dev/null 2>&1; then
 425                         echo "data/xonotic-maps.pk3dir does not exist, not downloading"
 426                         exit 0
 427                 fi
 428                 rundownload
 429                 cd ../..
 430                 echo "List of maps that got deleted (if any) and currently are in $bspdir.old:"
 431                 ls -l "$bspdir.old" || true
 432                 ;;
 433         download-latest)
 434                 mkdir -p "$bspdir"
 435                 cd "$bspdir"
 436                 rm -f *-????????????????????????????????????????-????????????????????????????????????????.pk3
 437                 wget -r -l1 -A "*.pk3" -N --no-parent --no-directories "$url_http""latest"
 438                 ;;
 439         log2spam-test)
 440                 log2spam "mapname" "http://mapurl" "branch" "commit" "0"
 441                 ;;
 442 esac