Merge branch 'fix-appdata-urls' into 'master'
[xonotic/xonotic.git] / misc / tools / xonotic-map-compiler-autobuild
1 #!/bin/sh
2
3 set -e
4
5 bspdir="$PWD/data"
6 url_http=https://beta.xonotic.org/autobuild-bsp/
7 url_ssh=xonotic-beta:./
8 build_cachedir="$HOME/xonotic-map-compiler.cache/"
9 screenshot_cachedir="$HOME/xonotic-map-screenshot.cache/"
10 build_override="-bsp_timeout 3600 -vis_timeout 10800 -light_timeout 18000 -minimap_timeout 900 -scale_timeout 900 -timeout_stealing 0.8"
11 screenshot_override="9 900 5 +g_ctf 1"
12
13 getthemap_fail=false
14
15 allmaps()
16 {
17         commithash=$1
18         shift
19         for F in `git show "${commithash}:maps/"`; do
20                 case "$F" in
21                         *.map.options)
22                                 ;;
23                         *)
24                                 continue
25                                 ;;
26                 esac
27                 M=${F%.map.options}
28                 blobhash=`git rev-parse --revs-only "${commithash}:maps/$M.map.options" || true`-`git rev-parse --revs-only "${commithash}:maps/$M.map" || true`
29                 case "$blobhash" in
30                         -*)
31                                 ;;
32                         *-)
33                                 ;;
34                         *)
35                                 "$@" "$M" "$blobhash" "$commithash"
36                                 ;;
37                 esac
38         done
39 }
40
41 pre2spam()
42 {
43         map=$1
44         url=$2
45         branch=$3
46         hash=$4
47
48         branch=${branch##refs/heads/}
49         branch=${branch##refs/remotes/}
50         branch=${branch##origin/}
51         hash=`echo "$hash" | cut -c 1-7`
52
53         echo "[$branch $hash] starting map compile of $map"
54 }
55
56 ss2spam()
57 {
58         map=$1
59         url=$2
60         branch=$3
61         hash=$4
62         status=$5
63
64         branch=${branch##refs/heads/}
65         branch=${branch##refs/remotes/}
66         branch=${branch##origin/}
67         hash=`echo "$hash" | cut -c 1-7`
68
69         s_error=
70
71         if [ "$status" -ne 0 ]; then
72                 s_error="exited with status $status"
73         fi
74
75         if [ -n "$s_error" ]; then
76                 echo "[$branch $hash] \ 34screenshots of $map could not be made: $s_error"
77         else
78                 echo "[$branch $hash] screenshots of $map are available on $url"
79         fi
80 }
81
82 log2spam()
83 {
84         map=$1
85         url=$2
86         branch=$3
87         hash=$4
88         status=$5
89         time=$6
90
91         hash=`echo "$hash" | cut -c 1-7`
92
93         branch=${branch##refs/heads/}
94         branch=${branch##refs/remotes/}
95         branch=${branch##origin/}
96
97         s_samplesize=
98         s_failshaders=
99         s_leaked=
100         s_error=
101
102         if [ "$status" -ne 0 ]; then
103                 s_error="exited with status $status"
104         fi
105         while IFS= read -r L; do
106                 case "$L" in
107                         WARNING:\ surface\ at\ *\ too\ large\ for\ desired\ samplesize*)
108                                 s=${L##* }
109                                 if [ -z "$s_samplesize" ] || [ "$s" -gt "$s_samplesize" ]; then
110                                         s_samplesize=$s
111                                 fi
112                                 ;;
113                         WARNING:\ Couldn\'t\ find\ image\ for\ shader\ textures/NULL)
114                                 # radiant stupid
115                                 ;;
116                         WARNING:\ Couldn\'t\ find\ image\ for\ shader\ *)
117                                 s_failshaders="$s_failshaders ${L##* }"
118                                 ;;
119                         ERROR:\ *)
120                                 if [ -z "$s_error" ]; then
121                                         s_error=$L
122                                 fi
123                                 ;;
124                         '******* leaked *******')
125                                 s_leaked=1
126                                 ;;
127                         '************ ERROR ************')
128                                 IFS= read -r s_error
129                                 ;;
130                 esac
131         done
132         s_failshaders=`echo "$s_failshaders" | sed "s, textures/, ,g"`
133         s_failshaders=${s_failshaders# }
134         if [ -n "$s_error" ]; then
135                 echo -n "[$branch $hash] \ 34failed"
136         else
137                 echo -n "[$branch $hash] finished"
138         fi
139         echo -n " map compile of $map ($url): $time sec"
140         if [ -n "$s_samplesize" ]; then
141                 echo -n ", FIX samplesize >= $s_samplesize"
142         fi
143         if [ -n "$s_failshaders" ]; then
144                 if [ -n "`echo "$s_failshaders" | cut -d ' ' -f 4-`" ]; then
145                         s_failshaders="`echo "$s_failshaders" | cut -d ' ' -f 1-3`..."
146                 fi
147                 echo -n ", FIX shaders $s_failshaders"
148         fi
149         if [ -n "$s_leaked" ]; then
150                 echo -n ", FIX LEAK"
151         fi
152         if [ -n "$s_error" ]; then
153                 s_error=`echo "$s_error" | sed "s,$PWD/\?,,g"`
154                 echo -n ", ERROR: $s_error"
155         fi
156         echo
157 }
158
159 buildthemap()
160 {
161         REFNAME=$1
162         url=$2
163         M=$3
164         blobhash=$4
165         commithash=$5
166         if lwp-request -m HEAD "$url$M-$blobhash.pk3"; then
167                 # already compiled
168                 return
169         fi
170
171         git reset --hard
172         git clean -xfd
173         git checkout -f "$commithash"
174
175         if [ -n "$IRCSPAM" ]; then
176                 pre2spam "$M" "$url$M-$blobhash.pk3" "$REFNAME" "$commithash" | $IRCSPAM || true
177         fi
178
179         t0=`date +%s`
180
181         # It's sad that we must enforce this rule by code.
182         if grep -- '-nosRGB' "maps/$M.map.options" >/dev/null; then
183                 is_nosrgb=true
184         else
185                 is_nosrgb=false
186         fi
187         allow_build=true
188         if $is_nosrgb; then
189                 case "$REFNAME" in
190                         master|refs/heads/master|origin/master|refs/remotes/origin/master)
191                                 if [ -n "$IRCSPAM" ]; then
192                                         echo "\ 34ERROR: map $M uses -nosRGB. This is not acceptable in master. Please fix." | $IRCSPAM || true
193                                 fi
194                                 allow_build=false
195                                 ;;
196                         *)
197                                 if [ -n "$IRCSPAM" ]; then
198                                         echo "\ 34WARNING:\ f map $M uses -nosRGB. This is not acceptable in master. Please fix." | $IRCSPAM || true
199                                 fi
200                                 ;;
201                 esac
202         fi
203
204         set +e
205         $allow_build && (
206                 cd ../..
207                 misc/tools/xonotic-map-compiler-optionsfile "data/xonotic-maps.pk3dir/maps/$M" $build_override > "data/xonotic-maps.pk3dir/maps/$M.log" 2>&1
208         )
209         status=$?
210         set -e
211         if ! [ -f "maps/$M.bsp" ]; then
212                 echo >>"maps/$M.log" "ERROR: No BSP file"
213         fi
214         t1=`date +%s`
215         dt=$(($t1 - $t0))
216         if [ -n "$IRCSPAM" ]; then
217                 cat "maps/$M.log" | log2spam "$M" "$url$M-$blobhash.pk3" "$REFNAME" "$commithash" "$status" "$dt" > "maps/$M.irc"
218         fi
219         zip -9r "$M-$blobhash.pk3" "maps/$M.bsp" "maps/$M.log" "maps/$M.irc" "maps/$M/" "maps/$M.lin" "gfx/${M}_mini.tga"
220         ln -snf "../$M-$blobhash.pk3" "$M.pk3" # from ALL branches, so beware!
221         cp "$M-$blobhash.pk3" "$M-full-$blobhash.pk3"
222         zip -9r "$M-full-$blobhash.pk3" `git diff --name-only --diff-filter=ACMRTUXB master...HEAD` || true
223         ln -snf "../$M-full-$blobhash.pk3" "$M-full.pk3" # from ALL branches, so beware!
224         rsync -vaSHP "$M-$blobhash.pk3" "$M-full-$blobhash.pk3" "$url_ssh"
225         rsync -vaSHP "$M.pk3" "$M-full.pk3" "$url_ssh""latest/"
226         if [ -n "$IRCSPAM" ]; then
227                 $IRCSPAM < "maps/$M.irc" || true
228         fi
229         if [ -n "$BUILD_ONE" ]; then
230                 exit
231         fi
232 }
233
234 screenshotthemap()
235 {
236         REFNAME=$1
237         url=$2
238         M=$3
239         blobhash=$4
240         commithash=$5
241         if ! [ -f "$bspdir/$M-$blobhash.pk3" ]; then
242                 # not downloaded yet
243                 return
244         fi
245
246         if lwp-request -m HEAD "$url$M-$blobhash/$M-000000.jpg"; then
247                 # already done
248                 return
249         fi
250
251         if ! unzip -l "$bspdir/$M-$blobhash.pk3" "maps/$M.bsp"; then
252                 # no BSP file
253                 return
254         fi
255
256         rm -rf ~/.xonotic/data
257         mkdir -p ~/.xonotic/data
258         (
259                 cd ../..
260                 if [ -n "$DISPLAY" ]; then
261                         misc/tools/xonotic-map-screenshot "$M" $screenshot_override $DRIVERFLAGS +"scr_screenshot_name \"$M-\""
262                 else
263                         case "$DRIVER" in
264                                 gl)
265                                         startx "$PWD/misc/tools/xonotic-map-screenshot" "$M" $screenshot_override $DRIVERFLAGS +"scr_screenshot_name \"$M-\"" -- :8
266                                         ;;
267                                 soft|*)
268                                         startx "$PWD/misc/tools/xonotic-map-screenshot" "$M" $screenshot_override +"vid_soft 1" $DRIVERFLAGS +"scr_screenshot_name \"$M-\"" -- /usr/bin/Xvfb :8 -screen 0 1024x768x24
269                                         ;;
270                         esac
271                 fi
272         ) 2>&1 | tee ~/.xonotic/data/autoscreenshot.log
273         good=false
274         if mv ~/.xonotic/data/screenshots "$M-$blobhash"; then
275                 ss2spam "$M" "$url$M-$blobhash/" "$REFNAME" "$commithash" 0 > ~/.xonotic/data/"$M.ircss"
276         else
277                 mkdir "$M-$blobhash"
278                 ss2spam "$M" "$url$M-$blobhash/" "$REFNAME" "$commithash" 1 > ~/.xonotic/data/"$M.ircss"
279         fi
280         mv ~/.xonotic/data/"$M.ircss" "$M-$blobhash"/
281         mv ~/.xonotic/data/autoscreenshot.log "$M-$blobhash"/
282         chmod 1777 "$M-$blobhash"
283         ln -snf "../$M-$blobhash" "$M" # from ALL branches, so beware!
284         rsync -vaSHP "$M-$blobhash" "$url_ssh"
285         rsync -vaSHP "$M" "$url_ssh""latest/"
286         if [ -n "$IRCSPAM" ]; then
287                 # It takes about 5 minutes for the server to update.
288                 sleep 360  # Cut it one more minute of slack.
289
290                 $IRCSPAM < "$M-$blobhash/$M.ircss" || true
291         fi
292         if [ -n "$BUILD_ONE" ]; then
293                 exit
294         fi
295 }
296
297 getthemap()
298 {
299         url=$1
300         bspdir_old=$2
301         bspdir=$3
302         M=$4
303         blobhash=$5
304         commithash=$6
305         if mv "$bspdir_old/$M-$blobhash.pk3" "$bspdir/$M-$blobhash.pk3"; then
306                 if unzip -l "$bspdir/$M-$blobhash.pk3" >/dev/null 2>&1; then
307                         return 0
308                 fi
309         fi
310         if ! wget -c -O "$bspdir/$M-$blobhash.pk3" "$url$M-$blobhash.pk3"; then
311                 if ! curl -Lo "$bspdir/$M-$blobhash.pk3" "$url$M-$blobhash.pk3"; then
312                         rm -f "$bspdir/$M-$blobhash.pk3"
313                         echo "WARNING: could not download $url$M-$blobhash.pk3, maybe not ready yet"
314                         getthemap_fail=true
315                         return 0
316                 fi
317         fi
318         if ! unzip -l "$bspdir/$M-$blobhash.pk3"; then
319                 rm -f "$bspdir/$M-$blobhash.pk3"
320                 echo "WARNING: could not download $url$M-$blobhash.pk3, invalid zip file"
321                 getthemap_fail=true
322                 return 0
323         fi
324 }
325
326 indexthemap()
327 {
328         REFNAME=$1
329         M=$2
330         blobhash=$3
331         commithash=$4
332         echo "$M $blobhash $commithash $REFNAME"
333 }
334
335 rundownload()
336 {
337         mkdir -p "$bspdir" "$bspdir.old"
338         for b in "$bspdir"/*-????????????????????????????????????????-????????????????????????????????????????.pk3; do
339                 if [ -e "$b" ]; then
340                         mv "$b" "$bspdir.old"/
341                 fi
342         done
343         allmaps "HEAD" getthemap "$url_http" "$bspdir.old" "$bspdir"
344 }
345
346 branches()
347 {
348         git for-each-ref 'refs/remotes' | grep -vE '    refs/remotes/([^/]*/HEAD|.*/archived/.*)$'
349 }
350
351 runmakeindex()
352 {
353         cd data/xonotic-maps.pk3dir
354         branches | while read -r HASH TYPE REFNAME; do
355                 allmaps "$HASH" indexthemap "$REFNAME"
356         done
357         cd ../..
358 }
359
360 case "$1" in
361         build)
362                 cd data/xonotic-maps.pk3dir
363                 branches | while read -r HASH TYPE REFNAME; do
364                         if [ -f "$build_cachedir/$HASH" ]; then
365                                 continue
366                         fi
367                         allmaps "$HASH" buildthemap "$REFNAME" "$url_http"
368                         touch "$build_cachedir/$HASH"
369                 done
370                 git checkout -f master
371                 ;;
372         screenshot)
373                 runmakeindex > "branches.idx.new"
374                 rsync -vaSHP "branches.idx.new" "$url_ssh""branches.idx.new"
375                 rm -f "branches.idx.new"
376                 cd data/xonotic-maps.pk3dir
377                 branches | while read -r HASH TYPE REFNAME; do
378                         if [ -f "$screenshot_cachedir/$HASH" ]; then
379                                 continue
380                         fi
381
382                         git reset --hard
383                         git clean -xfd
384                         git checkout -f "$HASH"
385
386                         getthemap_fail=false
387                         rundownload
388
389                         allmaps "$HASH" screenshotthemap "$REFNAME" "$url_http"
390
391                         if $getthemap_fail; then
392                                 # If any map fetch failed, we've skipped them and thus need to try again later.
393                                 # Note that we're not going to re-screenshot maps because of this as we'll check for uploaded screenshots first.
394                                 continue
395                         fi
396
397                         touch "$screenshot_cachedir/$HASH"
398                 done
399                 git checkout -f master
400                 ;;
401         makeindex)
402                 runmakeindex
403                 ;;
404         download)
405                 if ! which unzip >/dev/null 2>&1; then
406                         echo "Cannot find unzip, bailing out"
407                         exit 1
408                 fi
409                 if ! cd data/xonotic-maps.pk3dir >/dev/null 2>&1; then
410                         echo "data/xonotic-maps.pk3dir does not exist, not downloading"
411                         exit 0
412                 fi
413                 rundownload
414                 cd ../..
415                 echo "List of maps that got deleted (if any) and currently are in $bspdir.old:"
416                 ls -l "$bspdir.old" || true
417                 ;;
418         download-latest)
419                 mkdir -p "$bspdir"
420                 cd "$bspdir"
421                 rm -f *-????????????????????????????????????????-????????????????????????????????????????.pk3
422                 wget -r -l1 -A "*.pk3" -N --no-parent --no-directories "$url_http""latest"
423                 ;;
424         log2spam-test)
425                 log2spam "mapname" "http://mapurl" "branch" "commit" "0"
426                 ;;
427 esac