also upload that index
[xonotic/xonotic.git] / misc / tools / xonotic-map-compiler-autobuild
1 #!/bin/sh
2
3 set -e
4
5 bspdir="$PWD/data"
6 url_http=http://beta.xonotic.org/autobuild-bsp/
7 url_ssh=xonotic-beta:autobuild-bsp/
8 build_cachedir="$HOME/xonotic-map-compiler.cache/"
9 screenshot_cachedir="$HOME/xonotic-map-screenshot.cache/"
10 build_override="-bsp_timeout 900 -vis_timeout 3600 -light_timeout 14400 -minimap_timeout 900 -scale_timeout 900"
11 screenshot_override="9 600 +g_ctf 1"
12
13 getthemap_fail=false
14
15 allmaps()
16 {
17         commithash=$1
18         shift
19         for F in `git ls-files "$commithash" -- maps/\*.map.options`; do
20                 M=${F#maps/}
21                 M=${M%.map.options}
22                 blobhash=`git rev-parse --revs-only "$commithash:maps/$M.map.options" || true`-`git rev-parse --revs-only "$commithash:maps/$M.map" || true`
23                 case "$blobhash" in
24                         -*)
25                                 ;;
26                         *-)
27                                 ;;
28                         *)
29                                 "$@" "$M" "$blobhash" "$commithash"
30                                 ;;
31                 esac
32         done
33 }
34
35 pre2spam()
36 {
37         map=$1
38         url=$2
39         branch=$3
40         hash=$4
41
42         branch=${branch##refs/heads/}
43         branch=${branch##refs/remotes/}
44         branch=${branch##origin/}
45         hash=`echo "$hash" | cut -c 1-7`
46
47         echo "[$branch $hash] starting map compile of $map"
48 }
49
50 ss2spam()
51 {
52         map=$1
53         url=$2
54         branch=$3
55         hash=$4
56         status=$5
57
58         branch=${branch##refs/heads/}
59         branch=${branch##refs/remotes/}
60         branch=${branch##origin/}
61         hash=`echo "$hash" | cut -c 1-7`
62
63         s_error=
64
65         if [ "$status" -ne 0 ]; then
66                 s_error="exited with status $status"
67         fi
68
69         if [ -n "$s_error" ]; then
70                 echo "[$branch $hash] \ 34screenshots of $map could not be made: $s_error"
71         else
72                 echo "[$branch $hash] screenshots of $map are available on $url"
73         fi
74 }
75
76 log2spam()
77 {
78         map=$1
79         url=$2
80         branch=$3
81         hash=$4
82         status=$5
83         time=$6
84
85         hash=`echo "$hash" | cut -c 1-7`
86
87         branch=${branch##refs/heads/}
88         branch=${branch##refs/remotes/}
89         branch=${branch##origin/}
90
91         s_samplesize=
92         s_failshaders=
93         s_leaked=
94         s_error=
95
96         if [ "$status" -ne 0 ]; then
97                 s_error="exited with status $status"
98         fi
99         while IFS= read -r L; do
100                 case "$L" in
101                         WARNING:\ surface\ at\ *\ too\ large\ for\ desired\ samplesize*)
102                                 s=${L##* }
103                                 if [ -z "$s_samplesize" ] || [ "$s" -gt "$s_samplesize" ]; then
104                                         s_samplesize=$s
105                                 fi
106                                 ;;
107                         WARNING:\ Couldn\'t\ find\ image\ for\ shader\ textures/NULL)
108                                 # radiant stupid
109                                 ;;
110                         WARNING:\ Couldn\'t\ find\ image\ for\ shader\ *)
111                                 s_failshaders="$s_failshaders ${L##* }"
112                                 ;;
113                         '******* leaked *******')
114                                 s_leaked=1
115                                 ;;
116                         '************ ERROR ************')
117                                 IFS= read -r s_error
118                                 ;;
119                 esac
120         done
121         s_failshaders=`echo "$s_failshaders" | sed "s, textures/, ,g"`
122         s_failshaders=${s_failshaders# }
123         if [ -n "$s_error" ]; then
124                 echo -n "[$branch $hash] \ 34failed"
125         else
126                 echo -n "[$branch $hash] finished"
127         fi
128         echo -n " map compile of $map ($url): $time sec"
129         if [ -n "$s_samplesize" ]; then
130                 echo -n ", FIX samplesize >= $s_samplesize"
131         fi
132         if [ -n "$s_failshaders" ]; then
133                 if [ -n "`echo "$s_failshaders" | cut -d ' ' -f 4-`" ]; then
134                         s_failshaders="`echo "$s_failshaders" | cut -d ' ' -f 1-3`..."
135                 fi
136                 echo -n ", FIX shaders $s_failshaders"
137         fi
138         if [ -n "$s_leaked" ]; then
139                 echo -n ", FIX LEAK"
140         fi
141         if [ -n "$s_error" ]; then
142                 s_error=`echo "$s_error" | sed "s,$PWD/\?,,g"`
143                 echo -n ", ERROR: $s_error"
144         fi
145         echo
146 }
147
148 buildthemap()
149 {
150         REFNAME=$1
151         url=$2
152         M=$3
153         blobhash=$4
154         HASH=$5
155         if HEAD "$url$M-$blobhash.pk3"; then
156                 continue
157         fi
158
159         git reset --hard
160         git clean -xfd
161         git checkout -f "$HASH"
162
163         if [ -n "$IRCSPAM" ]; then
164                 pre2spam "$M" "$url$M-$blobhash.pk3" "$REFNAME" "$HASH" | $IRCSPAM
165         fi
166         t0=`date +%s`
167         (
168                 cd maps
169                 ../../../misc/tools/xonotic-map-compiler "$M" `grep ^- "$M.map.options" | cut -d '#' -f 1` $build_override > "$M.log"
170         )
171         t1=`date +%s`
172         dt=$(($t1 - $t0))
173         status=$?
174         if [ -n "$IRCSPAM" ]; then
175                 cat "maps/$M.log" | log2spam "$M" "$url$M-$blobhash.pk3" "$REFNAME" "$HASH" "$status" "$dt" > "maps/$M.irc"
176         fi
177         zip -9r "$M-$blobhash.pk3" "maps/$M.bsp" "maps/$M.log" "maps/$M.irc" "maps/$M/" "maps/$M.lin" "gfx/${M}_mini.tga"
178         ln -snf "../$M-$blobhash.pk3" "$M.pk3" # from ALL branches, so beware!
179         cp "$M-$blobhash.pk3" "$M-full-$blobhash.pk3"
180         zip -9r "$M-full-$blobhash.pk3" `git diff --name-only --diff-filter=ACMRTUXB master...HEAD` || true
181         ln -snf "../$M-full-$blobhash.pk3" "$M-full.pk3" # from ALL branches, so beware!
182         rsync -vaSHP "$M-$blobhash.pk3" "$M-full-$blobhash.pk3" "$url_ssh"
183         rsync -vaSHP "$M.pk3" "$M-full.pk3" "$url_ssh""latest/"
184         if [ -n "$IRCSPAM" ]; then
185                 $IRCSPAM < "maps/$M.irc"
186         fi
187 }
188
189 screenshotthemap()
190 {
191         REFNAME=$1
192         url=$2
193         M=$3
194         blobhash=$4
195         HASH=$5
196         if HEAD "$url$M-$blobhash/"; then
197                 continue
198         fi
199
200         git reset --hard
201         git clean -xfd
202         git checkout -f "$HASH"
203
204         rm -rf ~/.xonotic
205         (
206                 cd ../..
207                 if [ -n "$DISPLAY" ]; then
208                         misc/tools/xonotic-map-screenshot "$M" $screenshot_override +"scr_screenshot_name \"$M-\""
209                 else
210                         startx "$PWD/misc/tools/xonotic-map-screenshot" "$M" $screenshot_override +"scr_screenshot_name \"$M-\"" -- :8
211                 fi
212         )
213         if ! mv ~/.xonotic/data/screenshots "$M-$blobhash"; then
214                 if [ -n "$IRCSPAM" ]; then
215                         #ss2spam "$M" "$url$M-$blobhash/" "$REFNAME" "$HASH" 1 > "maps/$M.ircss"
216                         ss2spam "$M" "$url?d" "$REFNAME" "$HASH" 1 > "maps/$M.ircss"
217                 fi
218                 return 1
219         fi
220         if [ -n "$IRCSPAM" ]; then
221                 #ss2spam "$M" "$url$M-$blobhash/" "$REFNAME" "$HASH" 0 > "maps/$M.ircss"
222                 ss2spam "$M" "$url?d" "$REFNAME" "$HASH" 0 > "maps/$M.ircss"
223         fi
224         chmod 1777 "$M-$blobhash"
225         ln -snf "../$M-$blobhash" "$M" # from ALL branches, so beware!
226         rsync -vaSHP "$M-$blobhash" "$url_ssh"
227         rsync -vaSHP "$M" "$url_ssh""latest/"
228         if [ -n "$IRCSPAM" ]; then
229                 $IRCSPAM < "maps/$M.ircss"
230         fi
231 }
232
233 getthemap()
234 {
235         url=$1
236         bspdir_old=$2
237         bspdir=$3
238         M=$4
239         blobhash=$5
240         HASH=$6
241         if mv "$bspdir_old/$M-$blobhash.pk3" "$bspdir/$M-$blobhash.pk3"; then
242                 continue
243         fi
244         if ! wget -O "$bspdir/$M-$blobhash.pk3" "$url$M-$blobhash.pk3"; then
245                 if ! curl -o "$bspdir/$M-$blobhash.pk3" "$url$M-$blobhash.pk3"; then
246                         rm -f "$bspdir/$M-$blobhash.pk3"
247                         echo "WARNING: could not download $url$M-$blobhash.pk3, maybe not ready yet"
248                         getthemap_fail=true
249                         return 0
250                 fi
251         fi
252         if ! unzip -l "$bspdir/$M-$blobhash.pk3"; then
253                 rm -f "$bspdir/$M-$blobhash.pk3"
254                 echo "WARNING: could not download $url$M-$blobhash.pk3, invalid zip file"
255                 getthemap_fail=true
256                 return 0
257         fi
258 }
259
260 indexthemap()
261 {
262         REFNAME=$1
263         M=$2
264         blobhash=$3
265         HASH=$4
266         echo "$M $blobhash $HASH $REFNAME"
267 }
268
269 rundownload()
270 {
271         mkdir -p "$bspdir" "$bspdir.old"
272         for b in "$bspdir"/*-????????????????????????????????????????-????????????????????????????????????????.pk3; do
273                 if [ -e "$b" ]; then
274                         mv "$b" "$bspdir.old"/
275                 fi
276         done
277         cd data/xonotic-maps.pk3dir
278         allmaps "HEAD" getthemap "$url_http" "$bspdir.old" "$bspdir"
279         cd ../..
280 }
281
282 branches()
283 {
284         git for-each-ref 'refs/remotes' | grep -vE '    refs/remotes/([^/]*/HEAD|.*/archived/.*)$'
285 }
286
287 runmakeindex()
288 {
289         cd data/xonotic-maps.pk3dir
290         branches | while read -r HASH TYPE REFNAME; do
291                 allmaps "$HASH" indexthemap "$REFNAME"
292         done
293 }
294
295 case "$1" in
296         build)
297                 cd data/xonotic-maps.pk3dir
298                 branches | while read -r HASH TYPE REFNAME; do
299                         if [ -f "$build_cachedir/$HASH" ]; then
300                                 continue
301                         fi
302                         allmaps "$HASH" buildthemap "$REFNAME" "$url_http"
303                         touch "$build_cachedir/$HASH"
304                 done
305                 git checkout -f master
306                 ;;
307         screenshot)
308                 cd data/xonotic-maps.pk3dir
309                 branches | while read -r HASH TYPE REFNAME; do
310                         if [ -f "$screenshot_cachedir/$HASH" ]; then
311                                 continue
312                         fi
313
314                         rundownload
315                         if $getthemap_fail; then
316                                 continue
317                         fi
318
319                         cd data/xonotic-maps.pk3dir
320
321                         allmaps "$HASH" screenshotthemap "$REFNAME" "$url_http"
322                         touch "$screenshot_cachedir/$HASH"
323                 done
324                 git checkout -f master
325                 runmakeindex > branches.idx.new
326                 rsync -vaSHP "$branches.idx.new" "$url_ssh""branches.idx.new"
327                 rm -f branches.idx.new
328                 ;;
329         makeindex)
330                 runmakeindex
331                 ;;
332         download)
333                 rundownload
334                 echo "List of maps that got deleted (if any) and currently are in $bspdir.old:"
335                 ls -l "$bspdir.old"
336                 ;;
337         download-latest)
338                 mkdir -p "$bspdir"
339                 cd "$bspdir"
340                 rm -f *-????????????????????????????????????????-????????????????????????????????????????.pk3
341                 wget -r -l1 -A "*.pk3" -N --no-parent --no-directories "$url_http""latest"
342                 ;;
343         log2spam-test)
344                 log2spam "mapname" "http://mapurl" "branch" "commit" "0"
345                 ;;
346 esac