Merge branch 'master' into divVerent/crypto2
[xonotic/xonotic.git] / misc / tools / xonotic-map-compiler-autobuild
1 #!/bin/sh
2
3 set -e
4
5 bspdir="$PWD/data"
6 url_http=http://beta.xonotic.org/autobuild-bsp/
7 url_ssh=xonotic-beta:autobuild-bsp/
8 build_cachedir="$HOME/xonotic-map-compiler.cache/"
9 screenshot_cachedir="$HOME/xonotic-map-screenshot.cache/"
10 build_override="-bsp_timeout 900 -vis_timeout 3600 -light_timeout 14400 -minimap_timeout 900 -scale_timeout 900"
11 screenshot_override="9 600 +g_ctf 1"
12
13 getthemap_fail=false
14
15 allmaps()
16 {
17         commithash=$1
18         shift
19         for F in `git show "${commithash}:maps/"`; do
20                 case "$F" in
21                         *.map.options)
22                                 ;;
23                         *)
24                                 continue
25                                 ;;
26                 esac
27                 M=${F%.map.options}
28                 blobhash=`git rev-parse --revs-only "${commithash}:maps/$M.map.options" || true`-`git rev-parse --revs-only "${commithash}:maps/$M.map" || true`
29                 case "$blobhash" in
30                         -*)
31                                 ;;
32                         *-)
33                                 ;;
34                         *)
35                                 "$@" "$M" "$blobhash" "$commithash"
36                                 ;;
37                 esac
38         done
39 }
40
41 pre2spam()
42 {
43         map=$1
44         url=$2
45         branch=$3
46         hash=$4
47
48         branch=${branch##refs/heads/}
49         branch=${branch##refs/remotes/}
50         branch=${branch##origin/}
51         hash=`echo "$hash" | cut -c 1-7`
52
53         echo "[$branch $hash] starting map compile of $map"
54 }
55
56 ss2spam()
57 {
58         map=$1
59         url=$2
60         branch=$3
61         hash=$4
62         status=$5
63
64         branch=${branch##refs/heads/}
65         branch=${branch##refs/remotes/}
66         branch=${branch##origin/}
67         hash=`echo "$hash" | cut -c 1-7`
68
69         s_error=
70
71         if [ "$status" -ne 0 ]; then
72                 s_error="exited with status $status"
73         fi
74
75         if [ -n "$s_error" ]; then
76                 echo "[$branch $hash] \ 34screenshots of $map could not be made: $s_error"
77         else
78                 echo "[$branch $hash] screenshots of $map are available on $url"
79         fi
80 }
81
82 log2spam()
83 {
84         map=$1
85         url=$2
86         branch=$3
87         hash=$4
88         status=$5
89         time=$6
90
91         hash=`echo "$hash" | cut -c 1-7`
92
93         branch=${branch##refs/heads/}
94         branch=${branch##refs/remotes/}
95         branch=${branch##origin/}
96
97         s_samplesize=
98         s_failshaders=
99         s_leaked=
100         s_error=
101
102         if [ "$status" -ne 0 ]; then
103                 s_error="exited with status $status"
104         fi
105         while IFS= read -r L; do
106                 case "$L" in
107                         WARNING:\ surface\ at\ *\ too\ large\ for\ desired\ samplesize*)
108                                 s=${L##* }
109                                 if [ -z "$s_samplesize" ] || [ "$s" -gt "$s_samplesize" ]; then
110                                         s_samplesize=$s
111                                 fi
112                                 ;;
113                         WARNING:\ Couldn\'t\ find\ image\ for\ shader\ textures/NULL)
114                                 # radiant stupid
115                                 ;;
116                         WARNING:\ Couldn\'t\ find\ image\ for\ shader\ *)
117                                 s_failshaders="$s_failshaders ${L##* }"
118                                 ;;
119                         '******* leaked *******')
120                                 s_leaked=1
121                                 ;;
122                         '************ ERROR ************')
123                                 IFS= read -r s_error
124                                 ;;
125                 esac
126         done
127         s_failshaders=`echo "$s_failshaders" | sed "s, textures/, ,g"`
128         s_failshaders=${s_failshaders# }
129         if [ -n "$s_error" ]; then
130                 echo -n "[$branch $hash] \ 34failed"
131         else
132                 echo -n "[$branch $hash] finished"
133         fi
134         echo -n " map compile of $map ($url): $time sec"
135         if [ -n "$s_samplesize" ]; then
136                 echo -n ", FIX samplesize >= $s_samplesize"
137         fi
138         if [ -n "$s_failshaders" ]; then
139                 if [ -n "`echo "$s_failshaders" | cut -d ' ' -f 4-`" ]; then
140                         s_failshaders="`echo "$s_failshaders" | cut -d ' ' -f 1-3`..."
141                 fi
142                 echo -n ", FIX shaders $s_failshaders"
143         fi
144         if [ -n "$s_leaked" ]; then
145                 echo -n ", FIX LEAK"
146         fi
147         if [ -n "$s_error" ]; then
148                 s_error=`echo "$s_error" | sed "s,$PWD/\?,,g"`
149                 echo -n ", ERROR: $s_error"
150         fi
151         echo
152 }
153
154 buildthemap()
155 {
156         REFNAME=$1
157         url=$2
158         M=$3
159         blobhash=$4
160         commithash=$5
161         if HEAD "$url$M-$blobhash.pk3"; then
162                 continue
163         fi
164
165         git reset --hard
166         git clean -xfd
167         git checkout -f "$commithash"
168
169         if [ -n "$IRCSPAM" ]; then
170                 pre2spam "$M" "$url$M-$blobhash.pk3" "$REFNAME" "$commithash" | $IRCSPAM
171         fi
172         t0=`date +%s`
173         (
174                 cd maps
175                 sz=`grep '^size ' "$M.mapinfo" 2>/dev/null || true`
176                 if [ -n "$sz" ]; then
177                         minimap_override="-minimap + -minmax ${sz#size }"
178                 else
179                         minimap_override=
180                 fi
181                 ../../../misc/tools/xonotic-map-compiler "$M" `grep ^- "$M.map.options" | cut -d '#' -f 1` $minimap_override $build_override > "$M.log"
182         )
183         t1=`date +%s`
184         dt=$(($t1 - $t0))
185         status=$?
186         if [ -n "$IRCSPAM" ]; then
187                 cat "maps/$M.log" | log2spam "$M" "$url$M-$blobhash.pk3" "$REFNAME" "$commithash" "$status" "$dt" > "maps/$M.irc"
188         fi
189         zip -9r "$M-$blobhash.pk3" "maps/$M.bsp" "maps/$M.log" "maps/$M.irc" "maps/$M/" "maps/$M.lin" "gfx/${M}_mini.tga"
190         ln -snf "../$M-$blobhash.pk3" "$M.pk3" # from ALL branches, so beware!
191         cp "$M-$blobhash.pk3" "$M-full-$blobhash.pk3"
192         zip -9r "$M-full-$blobhash.pk3" `git diff --name-only --diff-filter=ACMRTUXB master...HEAD` || true
193         ln -snf "../$M-full-$blobhash.pk3" "$M-full.pk3" # from ALL branches, so beware!
194         rsync -vaSHP "$M-$blobhash.pk3" "$M-full-$blobhash.pk3" "$url_ssh"
195         rsync -vaSHP "$M.pk3" "$M-full.pk3" "$url_ssh""latest/"
196         if [ -n "$IRCSPAM" ]; then
197                 $IRCSPAM < "maps/$M.irc"
198         fi
199 }
200
201 screenshotthemap()
202 {
203         REFNAME=$1
204         url=$2
205         M=$3
206         blobhash=$4
207         commithash=$5
208         if HEAD "$url$M-$blobhash/"; then
209                 continue
210         fi
211
212         rm -rf ~/.xonotic
213         (
214                 cd ../..
215                 if [ -n "$DISPLAY" ]; then
216                         misc/tools/xonotic-map-screenshot "$M" $screenshot_override +"scr_screenshot_name \"$M-\""
217                 else
218                         startx "$PWD/misc/tools/xonotic-map-screenshot" "$M" $screenshot_override +"scr_screenshot_name \"$M-\"" -- :8
219                 fi
220         )
221         if ! mv ~/.xonotic/data/screenshots "$M-$blobhash"; then
222                 if [ -n "$IRCSPAM" ]; then
223                         #ss2spam "$M" "$url$M-$blobhash/" "$REFNAME" "$commithash" 1 > "maps/$M.ircss"
224                         ss2spam "$M" "$url?d" "$REFNAME" "$commithash" 1 > "maps/$M.ircss"
225                 fi
226                 return 1
227         fi
228         if [ -n "$IRCSPAM" ]; then
229                 #ss2spam "$M" "$url$M-$blobhash/" "$REFNAME" "$commithash" 0 > "maps/$M.ircss"
230                 ss2spam "$M" "$url?d" "$REFNAME" "$commithash" 0 > "maps/$M.ircss"
231         fi
232         chmod 1777 "$M-$blobhash"
233         ln -snf "../$M-$blobhash" "$M" # from ALL branches, so beware!
234         rsync -vaSHP "$M-$blobhash" "$url_ssh"
235         rsync -vaSHP "$M" "$url_ssh""latest/"
236         if [ -n "$IRCSPAM" ]; then
237                 $IRCSPAM < "maps/$M.ircss"
238         fi
239 }
240
241 getthemap()
242 {
243         url=$1
244         bspdir_old=$2
245         bspdir=$3
246         M=$4
247         blobhash=$5
248         commithash=$6
249         if mv "$bspdir_old/$M-$blobhash.pk3" "$bspdir/$M-$blobhash.pk3"; then
250                 continue
251         fi
252         if ! wget -O "$bspdir/$M-$blobhash.pk3" "$url$M-$blobhash.pk3"; then
253                 if ! curl -o "$bspdir/$M-$blobhash.pk3" "$url$M-$blobhash.pk3"; then
254                         rm -f "$bspdir/$M-$blobhash.pk3"
255                         echo "WARNING: could not download $url$M-$blobhash.pk3, maybe not ready yet"
256                         getthemap_fail=true
257                         return 0
258                 fi
259         fi
260         if ! unzip -l "$bspdir/$M-$blobhash.pk3"; then
261                 rm -f "$bspdir/$M-$blobhash.pk3"
262                 echo "WARNING: could not download $url$M-$blobhash.pk3, invalid zip file"
263                 getthemap_fail=true
264                 return 0
265         fi
266 }
267
268 indexthemap()
269 {
270         REFNAME=$1
271         M=$2
272         blobhash=$3
273         commithash=$4
274         echo "$M $blobhash $commithash $REFNAME"
275 }
276
277 rundownload()
278 {
279         mkdir -p "$bspdir" "$bspdir.old"
280         for b in "$bspdir"/*-????????????????????????????????????????-????????????????????????????????????????.pk3; do
281                 if [ -e "$b" ]; then
282                         mv "$b" "$bspdir.old"/
283                 fi
284         done
285         allmaps "HEAD" getthemap "$url_http" "$bspdir.old" "$bspdir"
286 }
287
288 branches()
289 {
290         git for-each-ref 'refs/remotes' | grep -vE '    refs/remotes/([^/]*/HEAD|.*/archived/.*)$'
291 }
292
293 runmakeindex()
294 {
295         cd data/xonotic-maps.pk3dir
296         branches | while read -r HASH TYPE REFNAME; do
297                 allmaps "$HASH" indexthemap "$REFNAME"
298         done
299         cd ../..
300 }
301
302 case "$1" in
303         build)
304                 cd data/xonotic-maps.pk3dir
305                 branches | while read -r HASH TYPE REFNAME; do
306                         if [ -f "$build_cachedir/$HASH" ]; then
307                                 continue
308                         fi
309                         allmaps "$HASH" buildthemap "$REFNAME" "$url_http"
310                         touch "$build_cachedir/$HASH"
311                 done
312                 git checkout -f master
313                 ;;
314         screenshot)
315                 runmakeindex > "branches.idx.new"
316                 rsync -vaSHP "branches.idx.new" "$url_ssh""branches.idx.new"
317                 rm -f "branches.idx.new"
318                 cd data/xonotic-maps.pk3dir
319                 branches | while read -r HASH TYPE REFNAME; do
320                         if [ -f "$screenshot_cachedir/$HASH" ]; then
321                                 continue
322                         fi
323
324                         git reset --hard
325                         git clean -xfd
326                         git checkout -f "$HASH"
327
328                         rundownload
329                         if $getthemap_fail; then
330                                 continue
331                         fi
332
333                         allmaps "$HASH" screenshotthemap "$REFNAME" "$url_http"
334
335                         touch "$screenshot_cachedir/$HASH"
336                 done
337                 git checkout -f master
338                 ;;
339         makeindex)
340                 runmakeindex
341                 ;;
342         download)
343                 cd data/xonotic-maps.pk3dir
344                 rundownload
345                 cd ../..
346                 echo "List of maps that got deleted (if any) and currently are in $bspdir.old:"
347                 ls -l "$bspdir.old" || true
348                 ;;
349         download-latest)
350                 mkdir -p "$bspdir"
351                 cd "$bspdir"
352                 rm -f *-????????????????????????????????????????-????????????????????????????????????????.pk3
353                 wget -r -l1 -A "*.pk3" -N --no-parent --no-directories "$url_http""latest"
354                 ;;
355         log2spam-test)
356                 log2spam "mapname" "http://mapurl" "branch" "commit" "0"
357                 ;;
358 esac