misc/tools/xonotic-map-compiler-autobuild

   1 #!/bin/sh
   2
   3 set -e
   4
   5 bspdir="$PWD/data"
   6 url_http=http://beta.xonotic.org/autobuild-bsp/
   7 url_ssh=xonotic-beta:autobuild-bsp/
   8 build_cachedir="$HOME/xonotic-map-compiler.cache/"
   9 screenshot_cachedir="$HOME/xonotic-map-screenshot.cache/"
  10 build_override="-bsp_timeout 900 -vis_timeout 3600 -light_timeout 14400 -minimap_timeout 900 -scale_timeout 900"
  11 screenshot_override="9 600 +g_ctf 1"
  12
  13 getthemap_fail=false
  14
  15 allmaps()
  16 {
  17         commithash=$1
  18         shift
  19         for F in `git show "${commithash}:maps/"`; do
  20                 case "$F" in
  21                         *.map.options)
  22                                 ;;
  23                         *)
  24                                 continue
  25                                 ;;
  26                 esac
  27                 M=${F%.map.options}
  28                 blobhash=`git rev-parse --revs-only "${commithash}:maps/$M.map.options" || true`-`git rev-parse --revs-only "${commithash}:maps/$M.map" || true`
  29                 case "$blobhash" in
  30                         -*)
  31                                 ;;
  32                         *-)
  33                                 ;;
  34                         *)
  35                                 "$@" "$M" "$blobhash" "$commithash"
  36                                 ;;
  37                 esac
  38         done
  39 }
  40
  41 pre2spam()
  42 {
  43         map=$1
  44         url=$2
  45         branch=$3
  46         hash=$4
  47
  48         branch=${branch##refs/heads/}
  49         branch=${branch##refs/remotes/}
  50         branch=${branch##origin/}
  51         hash=`echo "$hash" | cut -c 1-7`
  52
  53         echo "[$branch $hash] starting map compile of $map"
  54 }
  55
  56 ss2spam()
  57 {
  58         map=$1
  59         url=$2
  60         branch=$3
  61         hash=$4
  62         status=$5
  63
  64         branch=${branch##refs/heads/}
  65         branch=${branch##refs/remotes/}
  66         branch=${branch##origin/}
  67         hash=`echo "$hash" | cut -c 1-7`
  68
  69         s_error=
  70
  71         if [ "$status" -ne 0 ]; then
  72                 s_error="exited with status $status"
  73         fi
  74
  75         if [ -n "$s_error" ]; then
  76                 echo "[$branch $hash] \ 34screenshots of $map could not be made: $s_error"
  77         else
  78                 echo "[$branch $hash] screenshots of $map are available on $url"
  79         fi
  80 }
  81
  82 log2spam()
  83 {
  84         map=$1
  85         url=$2
  86         branch=$3
  87         hash=$4
  88         status=$5
  89         time=$6
  90
  91         hash=`echo "$hash" | cut -c 1-7`
  92
  93         branch=${branch##refs/heads/}
  94         branch=${branch##refs/remotes/}
  95         branch=${branch##origin/}
  96
  97         s_samplesize=
  98         s_failshaders=
  99         s_leaked=
 100         s_error=
 101
 102         if [ "$status" -ne 0 ]; then
 103                 s_error="exited with status $status"
 104         fi
 105         while IFS= read -r L; do
 106                 case "$L" in
 107                         WARNING:\ surface\ at\ *\ too\ large\ for\ desired\ samplesize*)
 108                                 s=${L##* }
 109                                 if [ -z "$s_samplesize" ] || [ "$s" -gt "$s_samplesize" ]; then
 110                                         s_samplesize=$s
 111                                 fi
 112                                 ;;
 113                         WARNING:\ Couldn\'t\ find\ image\ for\ shader\ textures/NULL)
 114                                 # radiant stupid
 115                                 ;;
 116                         WARNING:\ Couldn\'t\ find\ image\ for\ shader\ *)
 117                                 s_failshaders="$s_failshaders ${L##* }"
 118                                 ;;
 119                         '******* leaked *******')
 120                                 s_leaked=1
 121                                 ;;
 122                         '************ ERROR ************')
 123                                 IFS= read -r s_error
 124                                 ;;
 125                 esac
 126         done
 127         s_failshaders=`echo "$s_failshaders" | sed "s, textures/, ,g"`
 128         s_failshaders=${s_failshaders# }
 129         if [ -n "$s_error" ]; then
 130                 echo -n "[$branch $hash] \ 34failed"
 131         else
 132                 echo -n "[$branch $hash] finished"
 133         fi
 134         echo -n " map compile of $map ($url): $time sec"
 135         if [ -n "$s_samplesize" ]; then
 136                 echo -n ", FIX samplesize >= $s_samplesize"
 137         fi
 138         if [ -n "$s_failshaders" ]; then
 139                 if [ -n "`echo "$s_failshaders" | cut -d ' ' -f 4-`" ]; then
 140                         s_failshaders="`echo "$s_failshaders" | cut -d ' ' -f 1-3`..."
 141                 fi
 142                 echo -n ", FIX shaders $s_failshaders"
 143         fi
 144         if [ -n "$s_leaked" ]; then
 145                 echo -n ", FIX LEAK"
 146         fi
 147         if [ -n "$s_error" ]; then
 148                 s_error=`echo "$s_error" | sed "s,$PWD/\?,,g"`
 149                 echo -n ", ERROR: $s_error"
 150         fi
 151         echo
 152 }
 153
 154 buildthemap()
 155 {
 156         REFNAME=$1
 157         url=$2
 158         M=$3
 159         blobhash=$4
 160         commithash=$5
 161         if HEAD "$url$M-$blobhash.pk3"; then
 162                 continue
 163         fi
 164
 165         git reset --hard
 166         git clean -xfd
 167         git checkout -f "$commithash"
 168
 169         if [ -n "$IRCSPAM" ]; then
 170                 pre2spam "$M" "$url$M-$blobhash.pk3" "$REFNAME" "$commithash" | $IRCSPAM
 171         fi
 172         t0=`date +%s`
 173         (
 174                 cd maps
 175                 sz=`grep '^size ' "$M.mapinfo" 2>/dev/null || true`
 176                 if [ -n "$sz" ]; then
 177                         minimap_override="-minimap + -minmax ${sz#size }"
 178                 else
 179                         minimap_override=
 180                 fi
 181                 ../../../misc/tools/xonotic-map-compiler "$M" `grep ^- "$M.map.options" | cut -d '#' -f 1` $minimap_override $build_override > "$M.log"
 182         )
 183         t1=`date +%s`
 184         dt=$(($t1 - $t0))
 185         status=$?
 186         if [ -n "$IRCSPAM" ]; then
 187                 cat "maps/$M.log" | log2spam "$M" "$url$M-$blobhash.pk3" "$REFNAME" "$commithash" "$status" "$dt" > "maps/$M.irc"
 188         fi
 189         zip -9r "$M-$blobhash.pk3" "maps/$M.bsp" "maps/$M.log" "maps/$M.irc" "maps/$M/" "maps/$M.lin" "gfx/${M}_mini.tga"
 190         ln -snf "../$M-$blobhash.pk3" "$M.pk3" # from ALL branches, so beware!
 191         cp "$M-$blobhash.pk3" "$M-full-$blobhash.pk3"
 192         zip -9r "$M-full-$blobhash.pk3" `git diff --name-only --diff-filter=ACMRTUXB master...HEAD` || true
 193         ln -snf "../$M-full-$blobhash.pk3" "$M-full.pk3" # from ALL branches, so beware!
 194         rsync -vaSHP "$M-$blobhash.pk3" "$M-full-$blobhash.pk3" "$url_ssh"
 195         rsync -vaSHP "$M.pk3" "$M-full.pk3" "$url_ssh""latest/"
 196         if [ -n "$IRCSPAM" ]; then
 197                 $IRCSPAM < "maps/$M.irc"
 198         fi
 199 }
 200
 201 screenshotthemap()
 202 {
 203         REFNAME=$1
 204         url=$2
 205         M=$3
 206         blobhash=$4
 207         commithash=$5
 208         if HEAD "$url$M-$blobhash/"; then
 209                 continue
 210         fi
 211
 212         rm -rf ~/.xonotic
 213         (
 214                 cd ../..
 215                 if [ -n "$DISPLAY" ]; then
 216                         misc/tools/xonotic-map-screenshot "$M" $screenshot_override +"scr_screenshot_name \"$M-\""
 217                 else
 218                         startx "$PWD/misc/tools/xonotic-map-screenshot" "$M" $screenshot_override +"scr_screenshot_name \"$M-\"" -- :8
 219                 fi
 220         )
 221         if ! mv ~/.xonotic/data/screenshots "$M-$blobhash"; then
 222                 if [ -n "$IRCSPAM" ]; then
 223                         #ss2spam "$M" "$url$M-$blobhash/" "$REFNAME" "$commithash" 1 > "maps/$M.ircss"
 224                         ss2spam "$M" "$url?d" "$REFNAME" "$commithash" 1 > "maps/$M.ircss"
 225                 fi
 226                 return 1
 227         fi
 228         if [ -n "$IRCSPAM" ]; then
 229                 #ss2spam "$M" "$url$M-$blobhash/" "$REFNAME" "$commithash" 0 > "maps/$M.ircss"
 230                 ss2spam "$M" "$url?d" "$REFNAME" "$commithash" 0 > "maps/$M.ircss"
 231         fi
 232         chmod 1777 "$M-$blobhash"
 233         ln -snf "../$M-$blobhash" "$M" # from ALL branches, so beware!
 234         rsync -vaSHP "$M-$blobhash" "$url_ssh"
 235         rsync -vaSHP "$M" "$url_ssh""latest/"
 236         if [ -n "$IRCSPAM" ]; then
 237                 $IRCSPAM < "maps/$M.ircss"
 238         fi
 239 }
 240
 241 getthemap()
 242 {
 243         url=$1
 244         bspdir_old=$2
 245         bspdir=$3
 246         M=$4
 247         blobhash=$5
 248         commithash=$6
 249         if mv "$bspdir_old/$M-$blobhash.pk3" "$bspdir/$M-$blobhash.pk3"; then
 250                 continue
 251         fi
 252         if ! wget -O "$bspdir/$M-$blobhash.pk3" "$url$M-$blobhash.pk3"; then
 253                 if ! curl -o "$bspdir/$M-$blobhash.pk3" "$url$M-$blobhash.pk3"; then
 254                         rm -f "$bspdir/$M-$blobhash.pk3"
 255                         echo "WARNING: could not download $url$M-$blobhash.pk3, maybe not ready yet"
 256                         getthemap_fail=true
 257                         return 0
 258                 fi
 259         fi
 260         if ! unzip -l "$bspdir/$M-$blobhash.pk3"; then
 261                 rm -f "$bspdir/$M-$blobhash.pk3"
 262                 echo "WARNING: could not download $url$M-$blobhash.pk3, invalid zip file"
 263                 getthemap_fail=true
 264                 return 0
 265         fi
 266 }
 267
 268 indexthemap()
 269 {
 270         REFNAME=$1
 271         M=$2
 272         blobhash=$3
 273         commithash=$4
 274         echo "$M $blobhash $commithash $REFNAME"
 275 }
 276
 277 rundownload()
 278 {
 279         mkdir -p "$bspdir" "$bspdir.old"
 280         for b in "$bspdir"/*-????????????????????????????????????????-????????????????????????????????????????.pk3; do
 281                 if [ -e "$b" ]; then
 282                         mv "$b" "$bspdir.old"/
 283                 fi
 284         done
 285         allmaps "HEAD" getthemap "$url_http" "$bspdir.old" "$bspdir"
 286 }
 287
 288 branches()
 289 {
 290         git for-each-ref 'refs/remotes' | grep -vE '    refs/remotes/([^/]*/HEAD|.*/archived/.*)$'
 291 }
 292
 293 runmakeindex()
 294 {
 295         cd data/xonotic-maps.pk3dir
 296         branches | while read -r HASH TYPE REFNAME; do
 297                 allmaps "$HASH" indexthemap "$REFNAME"
 298         done
 299         cd ../..
 300 }
 301
 302 case "$1" in
 303         build)
 304                 cd data/xonotic-maps.pk3dir
 305                 branches | while read -r HASH TYPE REFNAME; do
 306                         if [ -f "$build_cachedir/$HASH" ]; then
 307                                 continue
 308                         fi
 309                         allmaps "$HASH" buildthemap "$REFNAME" "$url_http"
 310                         touch "$build_cachedir/$HASH"
 311                 done
 312                 git checkout -f master
 313                 ;;
 314         screenshot)
 315                 runmakeindex > "branches.idx.new"
 316                 rsync -vaSHP "branches.idx.new" "$url_ssh""branches.idx.new"
 317                 rm -f "branches.idx.new"
 318                 cd data/xonotic-maps.pk3dir
 319                 branches | while read -r HASH TYPE REFNAME; do
 320                         if [ -f "$screenshot_cachedir/$HASH" ]; then
 321                                 continue
 322                         fi
 323
 324                         git reset --hard
 325                         git clean -xfd
 326                         git checkout -f "$HASH"
 327
 328                         rundownload
 329                         if $getthemap_fail; then
 330                                 continue
 331                         fi
 332
 333                         allmaps "$HASH" screenshotthemap "$REFNAME" "$url_http"
 334
 335                         touch "$screenshot_cachedir/$HASH"
 336                 done
 337                 git checkout -f master
 338                 ;;
 339         makeindex)
 340                 runmakeindex
 341                 ;;
 342         download)
 343                 cd data/xonotic-maps.pk3dir
 344                 rundownload
 345                 cd ../..
 346                 echo "List of maps that got deleted (if any) and currently are in $bspdir.old:"
 347                 ls -l "$bspdir.old" || true
 348                 ;;
 349         download-latest)
 350                 mkdir -p "$bspdir"
 351                 cd "$bspdir"
 352                 rm -f *-????????????????????????????????????????-????????????????????????????????????????.pk3
 353                 wget -r -l1 -A "*.pk3" -N --no-parent --no-directories "$url_http""latest"
 354                 ;;
 355         log2spam-test)
 356                 log2spam "mapname" "http://mapurl" "branch" "commit" "0"
 357                 ;;
 358 esac