misc/tools/xonotic-map-compiler-autobuild

   1 #!/bin/sh
   2
   3 set -e
   4
   5 bspdir="$PWD/data"
   6 url_http=http://beta.xonotic.org/autobuild-bsp/
   7 url_ssh=xonotic-beta:autobuild-bsp/
   8 build_cachedir="$HOME/xonotic-map-compiler.cache/"
   9 screenshot_cachedir="$HOME/xonotic-map-screenshot.cache/"
  10 build_override="-bsp_timeout 900 -vis_timeout 3600 -light_timeout 14400 -minimap_timeout 900 -scale_timeout 900"
  11 screenshot_override="9 600 +g_ctf 1"
  12
  13 getthemap_fail=false
  14
  15 allmaps()
  16 {
  17         commithash=$1
  18         shift
  19         for F in `git ls-files "$commithash" -- maps/\*.map.options`; do
  20                 M=${F#maps/}
  21                 M=${M%.map.options}
  22                 blobhash=`git rev-parse --revs-only "$commithash:maps/$M.map.options" || true`-`git rev-parse --revs-only "$commithash:maps/$M.map" || true`
  23                 case "$blobhash" in
  24                         -*)
  25                                 ;;
  26                         *-)
  27                                 ;;
  28                         *)
  29                                 "$@" "$M" "$blobhash" "$commithash"
  30                                 ;;
  31                 esac
  32         done
  33 }
  34
  35 pre2spam()
  36 {
  37         map=$1
  38         url=$2
  39         branch=$3
  40         hash=$4
  41
  42         branch=${branch##refs/heads/}
  43         branch=${branch##refs/remotes/}
  44         branch=${branch##origin/}
  45         hash=`echo "$hash" | cut -c 1-7`
  46
  47         echo "[$branch $hash] starting map compile of $map"
  48 }
  49
  50 ss2spam()
  51 {
  52         map=$1
  53         url=$2
  54         branch=$3
  55         hash=$4
  56         status=$5
  57
  58         branch=${branch##refs/heads/}
  59         branch=${branch##refs/remotes/}
  60         branch=${branch##origin/}
  61         hash=`echo "$hash" | cut -c 1-7`
  62
  63         s_error=
  64
  65         if [ "$status" -ne 0 ]; then
  66                 s_error="exited with status $status"
  67         fi
  68
  69         if [ -n "$s_error" ]; then
  70                 echo "[$branch $hash] \ 34screenshots of $map could not be made: $s_error"
  71         else
  72                 echo "[$branch $hash] screenshots of $map are available on $url"
  73         fi
  74 }
  75
  76 log2spam()
  77 {
  78         map=$1
  79         url=$2
  80         branch=$3
  81         hash=$4
  82         status=$5
  83         time=$6
  84
  85         hash=`echo "$hash" | cut -c 1-7`
  86
  87         branch=${branch##refs/heads/}
  88         branch=${branch##refs/remotes/}
  89         branch=${branch##origin/}
  90
  91         s_samplesize=
  92         s_failshaders=
  93         s_leaked=
  94         s_error=
  95
  96         if [ "$status" -ne 0 ]; then
  97                 s_error="exited with status $status"
  98         fi
  99         while IFS= read -r L; do
 100                 case "$L" in
 101                         WARNING:\ surface\ at\ *\ too\ large\ for\ desired\ samplesize*)
 102                                 s=${L##* }
 103                                 if [ -z "$s_samplesize" ] || [ "$s" -gt "$s_samplesize" ]; then
 104                                         s_samplesize=$s
 105                                 fi
 106                                 ;;
 107                         WARNING:\ Couldn\'t\ find\ image\ for\ shader\ textures/NULL)
 108                                 # radiant stupid
 109                                 ;;
 110                         WARNING:\ Couldn\'t\ find\ image\ for\ shader\ *)
 111                                 s_failshaders="$s_failshaders ${L##* }"
 112                                 ;;
 113                         '******* leaked *******')
 114                                 s_leaked=1
 115                                 ;;
 116                         '************ ERROR ************')
 117                                 IFS= read -r s_error
 118                                 ;;
 119                 esac
 120         done
 121         s_failshaders=`echo "$s_failshaders" | sed "s, textures/, ,g"`
 122         s_failshaders=${s_failshaders# }
 123         if [ -n "$s_error" ]; then
 124                 echo -n "[$branch $hash] \ 34failed"
 125         else
 126                 echo -n "[$branch $hash] finished"
 127         fi
 128         echo -n " map compile of $map ($url): $time sec"
 129         if [ -n "$s_samplesize" ]; then
 130                 echo -n ", FIX samplesize >= $s_samplesize"
 131         fi
 132         if [ -n "$s_failshaders" ]; then
 133                 if [ -n "`echo "$s_failshaders" | cut -d ' ' -f 4-`" ]; then
 134                         s_failshaders="`echo "$s_failshaders" | cut -d ' ' -f 1-3`..."
 135                 fi
 136                 echo -n ", FIX shaders $s_failshaders"
 137         fi
 138         if [ -n "$s_leaked" ]; then
 139                 echo -n ", FIX LEAK"
 140         fi
 141         if [ -n "$s_error" ]; then
 142                 s_error=`echo "$s_error" | sed "s,$PWD/\?,,g"`
 143                 echo -n ", ERROR: $s_error"
 144         fi
 145         echo
 146 }
 147
 148 buildthemap()
 149 {
 150         REFNAME=$1
 151         url=$2
 152         M=$3
 153         blobhash=$4
 154         HASH=$5
 155         if HEAD "$url$M-$blobhash.pk3"; then
 156                 continue
 157         fi
 158
 159         git reset --hard
 160         git clean -xfd
 161         git checkout -f "$HASH"
 162
 163         if [ -n "$IRCSPAM" ]; then
 164                 pre2spam "$M" "$url$M-$blobhash.pk3" "$REFNAME" "$HASH" | $IRCSPAM
 165         fi
 166         t0=`date +%s`
 167         (
 168                 cd maps
 169                 ../../../misc/tools/xonotic-map-compiler "$M" `grep ^- "$M.map.options" | cut -d '#' -f 1` $build_override > "$M.log"
 170         )
 171         t1=`date +%s`
 172         dt=$(($t1 - $t0))
 173         status=$?
 174         if [ -n "$IRCSPAM" ]; then
 175                 cat "maps/$M.log" | log2spam "$M" "$url$M-$blobhash.pk3" "$REFNAME" "$HASH" "$status" "$dt" > "maps/$M.irc"
 176         fi
 177         zip -9r "$M-$blobhash.pk3" "maps/$M.bsp" "maps/$M.log" "maps/$M.irc" "maps/$M/" "maps/$M.lin" "gfx/${M}_mini.tga"
 178         ln -snf "../$M-$blobhash.pk3" "$M.pk3" # from ALL branches, so beware!
 179         cp "$M-$blobhash.pk3" "$M-full-$blobhash.pk3"
 180         zip -9r "$M-full-$blobhash.pk3" `git diff --name-only --diff-filter=ACMRTUXB master...HEAD` || true
 181         ln -snf "../$M-full-$blobhash.pk3" "$M-full.pk3" # from ALL branches, so beware!
 182         rsync -vaSHP "$M-$blobhash.pk3" "$M-full-$blobhash.pk3" "$url_ssh"
 183         rsync -vaSHP "$M.pk3" "$M-full.pk3" "$url_ssh""latest/"
 184         if [ -n "$IRCSPAM" ]; then
 185                 $IRCSPAM < "maps/$M.irc"
 186         fi
 187 }
 188
 189 screenshotthemap()
 190 {
 191         REFNAME=$1
 192         url=$2
 193         M=$3
 194         blobhash=$4
 195         HASH=$5
 196         if HEAD "$url$M-$blobhash/"; then
 197                 continue
 198         fi
 199
 200         git reset --hard
 201         git clean -xfd
 202         git checkout -f "$HASH"
 203
 204         rm -rf ~/.xonotic
 205         (
 206                 cd ../..
 207                 if [ -n "$DISPLAY" ]; then
 208                         misc/tools/xonotic-map-screenshot "$M" $screenshot_override +"scr_screenshot_name \"$M-\""
 209                 else
 210                         startx "$PWD/misc/tools/xonotic-map-screenshot" "$M" $screenshot_override +"scr_screenshot_name \"$M-\"" -- :8
 211                 fi
 212         )
 213         if ! mv ~/.xonotic/data/screenshots "$M-$blobhash"; then
 214                 if [ -n "$IRCSPAM" ]; then
 215                         #ss2spam "$M" "$url$M-$blobhash/" "$REFNAME" "$HASH" 1 > "maps/$M.ircss"
 216                         ss2spam "$M" "$url?d" "$REFNAME" "$HASH" 1 > "maps/$M.ircss"
 217                 fi
 218                 return 1
 219         fi
 220         if [ -n "$IRCSPAM" ]; then
 221                 #ss2spam "$M" "$url$M-$blobhash/" "$REFNAME" "$HASH" 0 > "maps/$M.ircss"
 222                 ss2spam "$M" "$url?d" "$REFNAME" "$HASH" 0 > "maps/$M.ircss"
 223         fi
 224         chmod 1777 "$M-$blobhash"
 225         ln -snf "../$M-$blobhash" "$M" # from ALL branches, so beware!
 226         rsync -vaSHP "$M-$blobhash" "$url_ssh"
 227         rsync -vaSHP "$M" "$url_ssh""latest/"
 228         if [ -n "$IRCSPAM" ]; then
 229                 $IRCSPAM < "maps/$M.ircss"
 230         fi
 231 }
 232
 233 getthemap()
 234 {
 235         url=$1
 236         bspdir_old=$2
 237         bspdir=$3
 238         M=$4
 239         blobhash=$5
 240         HASH=$6
 241         if mv "$bspdir_old/$M-$blobhash.pk3" "$bspdir/$M-$blobhash.pk3"; then
 242                 continue
 243         fi
 244         if ! wget -O "$bspdir/$M-$blobhash.pk3" "$url$M-$blobhash.pk3"; then
 245                 if ! curl -o "$bspdir/$M-$blobhash.pk3" "$url$M-$blobhash.pk3"; then
 246                         rm -f "$bspdir/$M-$blobhash.pk3"
 247                         echo "WARNING: could not download $url$M-$blobhash.pk3, maybe not ready yet"
 248                         getthemap_fail=true
 249                         return 0
 250                 fi
 251         fi
 252         if ! unzip -l "$bspdir/$M-$blobhash.pk3"; then
 253                 rm -f "$bspdir/$M-$blobhash.pk3"
 254                 echo "WARNING: could not download $url$M-$blobhash.pk3, invalid zip file"
 255                 getthemap_fail=true
 256                 return 0
 257         fi
 258 }
 259
 260 indexthemap()
 261 {
 262         REFNAME=$1
 263         M=$2
 264         blobhash=$3
 265         HASH=$4
 266         echo "$M $blobhash $HASH $REFNAME"
 267 }
 268
 269 rundownload()
 270 {
 271         mkdir -p "$bspdir" "$bspdir.old"
 272         for b in "$bspdir"/*-????????????????????????????????????????-????????????????????????????????????????.pk3; do
 273                 if [ -e "$b" ]; then
 274                         mv "$b" "$bspdir.old"/
 275                 fi
 276         done
 277         cd data/xonotic-maps.pk3dir
 278         allmaps "HEAD" getthemap "$url_http" "$bspdir.old" "$bspdir"
 279         cd ../..
 280 }
 281
 282 branches()
 283 {
 284         git for-each-ref 'refs/remotes' | grep -vE '    refs/remotes/([^/]*/HEAD|.*/archived/.*)$'
 285 }
 286
 287 runmakeindex()
 288 {
 289         cd data/xonotic-maps.pk3dir
 290         branches | while read -r HASH TYPE REFNAME; do
 291                 allmaps "$HASH" indexthemap "$REFNAME"
 292         done
 293 }
 294
 295 case "$1" in
 296         build)
 297                 cd data/xonotic-maps.pk3dir
 298                 branches | while read -r HASH TYPE REFNAME; do
 299                         if [ -f "$build_cachedir/$HASH" ]; then
 300                                 continue
 301                         fi
 302                         allmaps "$HASH" buildthemap "$REFNAME" "$url_http"
 303                         touch "$build_cachedir/$HASH"
 304                 done
 305                 git checkout -f master
 306                 ;;
 307         screenshot)
 308                 cd data/xonotic-maps.pk3dir
 309                 runmakeindex > branches.idx.new
 310                 rsync -vaSHP "$branches.idx.new" "$url_ssh""branches.idx.new"
 311                 rm -f branches.idx.new
 312                 branches | while read -r HASH TYPE REFNAME; do
 313                         if [ -f "$screenshot_cachedir/$HASH" ]; then
 314                                 continue
 315                         fi
 316
 317                         rundownload
 318                         if $getthemap_fail; then
 319                                 continue
 320                         fi
 321
 322                         cd data/xonotic-maps.pk3dir
 323                         allmaps "$HASH" screenshotthemap "$REFNAME" "$url_http"
 324                         cd ../..
 325
 326                         touch "$screenshot_cachedir/$HASH"
 327                 done
 328                 git checkout -f master
 329                 ;;
 330         makeindex)
 331                 runmakeindex
 332                 ;;
 333         download)
 334                 rundownload
 335                 echo "List of maps that got deleted (if any) and currently are in $bspdir.old:"
 336                 ls -l "$bspdir.old" || true
 337                 ;;
 338         download-latest)
 339                 mkdir -p "$bspdir"
 340                 cd "$bspdir"
 341                 rm -f *-????????????????????????????????????????-????????????????????????????????????????.pk3
 342                 wget -r -l1 -A "*.pk3" -N --no-parent --no-directories "$url_http""latest"
 343                 ;;
 344         log2spam-test)
 345                 log2spam "mapname" "http://mapurl" "branch" "commit" "0"
 346                 ;;
 347 esac