misc/tools/xonotic-map-compiler-autobuild

   1 #!/bin/sh
   2
   3 set -e
   4
   5 bspdir="$PWD/data"
   6 url_http=http://beta.xonotic.org/autobuild-bsp/
   7 url_ssh=xonotic-beta:autobuild-bsp/
   8 build_cachedir="$HOME/xonotic-map-compiler.cache/"
   9 screenshot_cachedir="$HOME/xonotic-map-screenshot.cache/"
  10 build_override="-bsp_timeout 3600 -vis_timeout 10800 -light_timeout 18000 -minimap_timeout 900 -scale_timeout 900 -timeout_stealing 0.8"
  11 screenshot_override="9 900 5 +g_ctf 1"
  12
  13 getthemap_fail=false
  14
  15 allmaps()
  16 {
  17         commithash=$1
  18         shift
  19         for F in `git show "${commithash}:maps/"`; do
  20                 case "$F" in
  21                         *.map.options)
  22                                 ;;
  23                         *)
  24                                 continue
  25                                 ;;
  26                 esac
  27                 M=${F%.map.options}
  28                 blobhash=`git rev-parse --revs-only "${commithash}:maps/$M.map.options" || true`-`git rev-parse --revs-only "${commithash}:maps/$M.map" || true`
  29                 case "$blobhash" in
  30                         -*)
  31                                 ;;
  32                         *-)
  33                                 ;;
  34                         *)
  35                                 "$@" "$M" "$blobhash" "$commithash"
  36                                 ;;
  37                 esac
  38         done
  39 }
  40
  41 pre2spam()
  42 {
  43         map=$1
  44         url=$2
  45         branch=$3
  46         hash=$4
  47
  48         branch=${branch##refs/heads/}
  49         branch=${branch##refs/remotes/}
  50         branch=${branch##origin/}
  51         hash=`echo "$hash" | cut -c 1-7`
  52
  53         echo "[$branch $hash] starting map compile of $map"
  54 }
  55
  56 ss2spam()
  57 {
  58         map=$1
  59         url=$2
  60         branch=$3
  61         hash=$4
  62         status=$5
  63
  64         branch=${branch##refs/heads/}
  65         branch=${branch##refs/remotes/}
  66         branch=${branch##origin/}
  67         hash=`echo "$hash" | cut -c 1-7`
  68
  69         s_error=
  70
  71         if [ "$status" -ne 0 ]; then
  72                 s_error="exited with status $status"
  73         fi
  74
  75         if [ -n "$s_error" ]; then
  76                 echo "[$branch $hash] \ 34screenshots of $map could not be made: $s_error"
  77         else
  78                 echo "[$branch $hash] screenshots of $map are available on $url"
  79         fi
  80 }
  81
  82 log2spam()
  83 {
  84         map=$1
  85         url=$2
  86         branch=$3
  87         hash=$4
  88         status=$5
  89         time=$6
  90
  91         hash=`echo "$hash" | cut -c 1-7`
  92
  93         branch=${branch##refs/heads/}
  94         branch=${branch##refs/remotes/}
  95         branch=${branch##origin/}
  96
  97         s_samplesize=
  98         s_failshaders=
  99         s_leaked=
 100         s_error=
 101
 102         if [ "$status" -ne 0 ]; then
 103                 s_error="exited with status $status"
 104         fi
 105         while IFS= read -r L; do
 106                 case "$L" in
 107                         WARNING:\ surface\ at\ *\ too\ large\ for\ desired\ samplesize*)
 108                                 s=${L##* }
 109                                 if [ -z "$s_samplesize" ] || [ "$s" -gt "$s_samplesize" ]; then
 110                                         s_samplesize=$s
 111                                 fi
 112                                 ;;
 113                         WARNING:\ Couldn\'t\ find\ image\ for\ shader\ textures/NULL)
 114                                 # radiant stupid
 115                                 ;;
 116                         WARNING:\ Couldn\'t\ find\ image\ for\ shader\ *)
 117                                 s_failshaders="$s_failshaders ${L##* }"
 118                                 ;;
 119                         ERROR:\ *)
 120                                 if [ -z "$s_error" ]; then
 121                                         s_error=$L
 122                                 fi
 123                                 ;;
 124                         '******* leaked *******')
 125                                 s_leaked=1
 126                                 ;;
 127                         '************ ERROR ************')
 128                                 IFS= read -r s_error
 129                                 ;;
 130                 esac
 131         done
 132         s_failshaders=`echo "$s_failshaders" | sed "s, textures/, ,g"`
 133         s_failshaders=${s_failshaders# }
 134         if [ -n "$s_error" ]; then
 135                 echo -n "[$branch $hash] \ 34failed"
 136         else
 137                 echo -n "[$branch $hash] finished"
 138         fi
 139         echo -n " map compile of $map ($url): $time sec"
 140         if [ -n "$s_samplesize" ]; then
 141                 echo -n ", FIX samplesize >= $s_samplesize"
 142         fi
 143         if [ -n "$s_failshaders" ]; then
 144                 if [ -n "`echo "$s_failshaders" | cut -d ' ' -f 4-`" ]; then
 145                         s_failshaders="`echo "$s_failshaders" | cut -d ' ' -f 1-3`..."
 146                 fi
 147                 echo -n ", FIX shaders $s_failshaders"
 148         fi
 149         if [ -n "$s_leaked" ]; then
 150                 echo -n ", FIX LEAK"
 151         fi
 152         if [ -n "$s_error" ]; then
 153                 s_error=`echo "$s_error" | sed "s,$PWD/\?,,g"`
 154                 echo -n ", ERROR: $s_error"
 155         fi
 156         echo
 157 }
 158
 159 buildthemap()
 160 {
 161         REFNAME=$1
 162         url=$2
 163         M=$3
 164         blobhash=$4
 165         commithash=$5
 166         if lwp-request -m HEAD "$url$M-$blobhash.pk3"; then
 167                 continue
 168         fi
 169
 170         git reset --hard
 171         git clean -xfd
 172         git checkout -f "$commithash"
 173
 174         if [ -n "$IRCSPAM" ]; then
 175                 pre2spam "$M" "$url$M-$blobhash.pk3" "$REFNAME" "$commithash" | $IRCSPAM || true
 176         fi
 177
 178         t0=`date +%s`
 179
 180         # It's sad that we must enforce this rule by code.
 181         if grep -- '-nosRGB' "maps/$M.map.options" >/dev/null; then
 182                 is_nosrgb=true
 183         else
 184                 is_nosrgb=false
 185         fi
 186         allow_build=true
 187         case "$M" in
 188                 afterslime|dance|drain|glowplant|leave_em_behind|newtonian-nightmare|nexballarena|oilrig|red-planet|runningmanctf|runningman|space-elevator|stormkeep|techassault|xoylent)
 189                         if $is_nosrgb; then
 190                                 if [ -n "$IRCSPAM" ]; then
 191                                         echo "\ 38NOTE:\ f map $M still uses -nosRGB. This should eventually get fixed." | $IRCSPAM || true
 192                                 fi
 193                         else
 194                                 if [ -n "$IRCSPAM" ]; then
 195                                         echo "\ 39CONGRATULATIONS: map $M does not use -nosRGB any more. Please remove the map name from the list in xonotic-map-compiler-autobuild once this is merged to master." | $IRCSPAM || true
 196                                 fi
 197                         fi
 198                         ;;
 199                 *)
 200                         if $is_nosrgb; then
 201                                 case "$REFNAME" in
 202                                         master|refs/heads/master|origin/master|refs/remotes/origin/master)
 203                                                 if [ -n "$IRCSPAM" ]; then
 204                                                         echo "\ 34ERROR: map $M uses -nosRGB. This is not acceptable in master. Please fix." | $IRCSPAM || true
 205                                                 fi
 206                                                 allow_build=false
 207                                                 ;;
 208                                         *)
 209                                                 if [ -n "$IRCSPAM" ]; then
 210                                                         echo "\ 34WARNING:\ f map $M uses -nosRGB. This is not acceptable in master. Please fix." | $IRCSPAM || true
 211                                                 fi
 212                                                 ;;
 213                                 esac
 214                         fi
 215                         ;;
 216         esac
 217
 218         set +e
 219         $allow_build && (
 220                 cd ../..
 221                 misc/tools/xonotic-map-compiler-optionsfile "data/xonotic-maps.pk3dir/maps/$M" $build_override > "data/xonotic-maps.pk3dir/maps/$M.log" 2>&1
 222         )
 223         status=$?
 224         set -e
 225         if ! [ -f "maps/$M.bsp" ]; then
 226                 echo >>"maps/$M.log" "ERROR: No BSP file"
 227         fi
 228         t1=`date +%s`
 229         dt=$(($t1 - $t0))
 230         if [ -n "$IRCSPAM" ]; then
 231                 cat "maps/$M.log" | log2spam "$M" "$url$M-$blobhash.pk3" "$REFNAME" "$commithash" "$status" "$dt" > "maps/$M.irc"
 232         fi
 233         zip -9r "$M-$blobhash.pk3" "maps/$M.bsp" "maps/$M.log" "maps/$M.irc" "maps/$M/" "maps/$M.lin" "gfx/${M}_mini.tga"
 234         ln -snf "../$M-$blobhash.pk3" "$M.pk3" # from ALL branches, so beware!
 235         cp "$M-$blobhash.pk3" "$M-full-$blobhash.pk3"
 236         zip -9r "$M-full-$blobhash.pk3" `git diff --name-only --diff-filter=ACMRTUXB master...HEAD` || true
 237         ln -snf "../$M-full-$blobhash.pk3" "$M-full.pk3" # from ALL branches, so beware!
 238         rsync -vaSHP "$M-$blobhash.pk3" "$M-full-$blobhash.pk3" "$url_ssh"
 239         rsync -vaSHP "$M.pk3" "$M-full.pk3" "$url_ssh""latest/"
 240         if [ -n "$IRCSPAM" ]; then
 241                 $IRCSPAM < "maps/$M.irc" || true
 242         fi
 243         if [ -n "$BUILD_ONE" ]; then
 244                 exit
 245         fi
 246 }
 247
 248 screenshotthemap()
 249 {
 250         REFNAME=$1
 251         url=$2
 252         M=$3
 253         blobhash=$4
 254         commithash=$5
 255         if lwp-request -m HEAD "$url$M-$blobhash/$M-000000.jpg"; then
 256                 continue
 257         fi
 258
 259         if ! unzip -l "$bspdir/$M-$blobhash.pk3" "maps/$M.bsp"; then
 260                 # no BSP file
 261                 continue
 262         fi
 263
 264         rm -rf ~/.xonotic/data
 265         mkdir -p ~/.xonotic/data
 266         (
 267                 cd ../..
 268                 if [ -n "$DISPLAY" ]; then
 269                         misc/tools/xonotic-map-screenshot "$M" $screenshot_override $DRIVERFLAGS +"scr_screenshot_name \"$M-\""
 270                 else
 271                         case "$DRIVER" in
 272                                 gl)
 273                                         startx "$PWD/misc/tools/xonotic-map-screenshot" "$M" $screenshot_override $DRIVERFLAGS +"scr_screenshot_name \"$M-\"" -- :8
 274                                         ;;
 275                                 soft|*)
 276                                         startx "$PWD/misc/tools/xonotic-map-screenshot" "$M" $screenshot_override +"vid_soft 1" $DRIVERFLAGS +"scr_screenshot_name \"$M-\"" -- /usr/bin/Xvfb :8 -screen 0 1024x768x24
 277                                         ;;
 278                         esac
 279                 fi
 280         ) 2>&1 | tee ~/.xonotic/data/autoscreenshot.log
 281         good=false
 282         if mv ~/.xonotic/data/screenshots "$M-$blobhash"; then
 283                 ss2spam "$M" "$url?d" "$REFNAME" "$commithash" 0 > ~/.xonotic/data/"$M.ircss"
 284         else
 285                 mkdir "$M-$blobhash"
 286                 ss2spam "$M" "$url?d" "$REFNAME" "$commithash" 1 > ~/.xonotic/data/"$M.ircss"
 287         fi
 288         mv ~/.xonotic/data/"$M.ircss" "$M-$blobhash"/
 289         mv ~/.xonotic/data/autoscreenshot.log "$M-$blobhash"/
 290         chmod 1777 "$M-$blobhash"
 291         ln -snf "../$M-$blobhash" "$M" # from ALL branches, so beware!
 292         rsync -vaSHP "$M-$blobhash" "$url_ssh"
 293         rsync -vaSHP "$M" "$url_ssh""latest/"
 294         if [ -n "$IRCSPAM" ]; then
 295                 # It takes about 5 minutes for the server to update.
 296                 sleep 360  # Cut it one more minute of slack.
 297
 298                 $IRCSPAM < "$M-$blobhash/$M.ircss" || true
 299         fi
 300         if [ -n "$BUILD_ONE" ]; then
 301                 exit
 302         fi
 303 }
 304
 305 getthemap()
 306 {
 307         url=$1
 308         bspdir_old=$2
 309         bspdir=$3
 310         M=$4
 311         blobhash=$5
 312         commithash=$6
 313         if mv "$bspdir_old/$M-$blobhash.pk3" "$bspdir/$M-$blobhash.pk3"; then
 314                 if unzip -l "$bspdir/$M-$blobhash.pk3" >/dev/null 2>&1; then
 315                         return 0
 316                 fi
 317         fi
 318         if ! wget -c -O "$bspdir/$M-$blobhash.pk3" "$url$M-$blobhash.pk3"; then
 319                 if ! curl -o "$bspdir/$M-$blobhash.pk3" "$url$M-$blobhash.pk3"; then
 320                         rm -f "$bspdir/$M-$blobhash.pk3"
 321                         echo "WARNING: could not download $url$M-$blobhash.pk3, maybe not ready yet"
 322                         getthemap_fail=true
 323                         return 0
 324                 fi
 325         fi
 326         if ! unzip -l "$bspdir/$M-$blobhash.pk3"; then
 327                 rm -f "$bspdir/$M-$blobhash.pk3"
 328                 echo "WARNING: could not download $url$M-$blobhash.pk3, invalid zip file"
 329                 getthemap_fail=true
 330                 return 0
 331         fi
 332 }
 333
 334 indexthemap()
 335 {
 336         REFNAME=$1
 337         M=$2
 338         blobhash=$3
 339         commithash=$4
 340         echo "$M $blobhash $commithash $REFNAME"
 341 }
 342
 343 rundownload()
 344 {
 345         mkdir -p "$bspdir" "$bspdir.old"
 346         for b in "$bspdir"/*-????????????????????????????????????????-????????????????????????????????????????.pk3; do
 347                 if [ -e "$b" ]; then
 348                         mv "$b" "$bspdir.old"/
 349                 fi
 350         done
 351         allmaps "HEAD" getthemap "$url_http" "$bspdir.old" "$bspdir"
 352 }
 353
 354 branches()
 355 {
 356         git for-each-ref 'refs/remotes' | grep -vE '    refs/remotes/([^/]*/HEAD|.*/archived/.*)$'
 357 }
 358
 359 runmakeindex()
 360 {
 361         cd data/xonotic-maps.pk3dir
 362         branches | while read -r HASH TYPE REFNAME; do
 363                 allmaps "$HASH" indexthemap "$REFNAME"
 364         done
 365         cd ../..
 366 }
 367
 368 case "$1" in
 369         build)
 370                 cd data/xonotic-maps.pk3dir
 371                 branches | while read -r HASH TYPE REFNAME; do
 372                         if [ -f "$build_cachedir/$HASH" ]; then
 373                                 continue
 374                         fi
 375                         allmaps "$HASH" buildthemap "$REFNAME" "$url_http"
 376                         touch "$build_cachedir/$HASH"
 377                 done
 378                 git checkout -f master
 379                 ;;
 380         screenshot)
 381                 runmakeindex > "branches.idx.new"
 382                 rsync -vaSHP "branches.idx.new" "$url_ssh""branches.idx.new"
 383                 rm -f "branches.idx.new"
 384                 cd data/xonotic-maps.pk3dir
 385                 branches | while read -r HASH TYPE REFNAME; do
 386                         if [ -f "$screenshot_cachedir/$HASH" ]; then
 387                                 continue
 388                         fi
 389
 390                         git reset --hard
 391                         git clean -xfd
 392                         git checkout -f "$HASH"
 393
 394                         rundownload
 395                         if $getthemap_fail; then
 396                                 continue
 397                         fi
 398
 399                         allmaps "$HASH" screenshotthemap "$REFNAME" "$url_http"
 400
 401                         touch "$screenshot_cachedir/$HASH"
 402                 done
 403                 git checkout -f master
 404                 ;;
 405         makeindex)
 406                 runmakeindex
 407                 ;;
 408         download)
 409                 if ! which unzip >/dev/null 2>&1; then
 410                         echo "Cannot find unzip, bailing out"
 411                         exit 1
 412                 fi
 413                 if ! cd data/xonotic-maps.pk3dir >/dev/null 2>&1; then
 414                         echo "data/xonotic-maps.pk3dir does not exist, not downloading"
 415                         exit 0
 416                 fi
 417                 rundownload
 418                 cd ../..
 419                 echo "List of maps that got deleted (if any) and currently are in $bspdir.old:"
 420                 ls -l "$bspdir.old" || true
 421                 ;;
 422         download-latest)
 423                 mkdir -p "$bspdir"
 424                 cd "$bspdir"
 425                 rm -f *-????????????????????????????????????????-????????????????????????????????????????.pk3
 426                 wget -r -l1 -A "*.pk3" -N --no-parent --no-directories "$url_http""latest"
 427                 ;;
 428         log2spam-test)
 429                 log2spam "mapname" "http://mapurl" "branch" "commit" "0"
 430                 ;;
 431 esac