clean up pk3s properly
[xonotic/xonotic.git] / misc / tools / xonotic-map-compiler-autobuild
1 #!/bin/sh
2
3 set -e
4
5 bspdir="$PWD/data"
6 http_url=http://beta.xonotic.org/autobuild-bsp/
7 url_ssh=xonotic-beta:autobuild-bsp/
8 build_cachedir="$HOME/xonotic-map-compiler.cache/"
9 screenshot_cachedir="$HOME/xonotic-map-screenshot.cache/"
10 build_override="-bsp_timeout 900 -vis_timeout 3600 -light_timeout 14400 -minimap_timeout 900 -scale_timeout 900"
11 screenshot_override="9 300"
12
13 getthemap_fail=false
14
15 allmaps()
16 {
17         for F in maps/*.map.options; do
18                 if ! [ -f "$F" ]; then
19                         continue
20                 fi
21                 if ! [ -f "${F%.options}" ]; then
22                         continue
23                 fi
24                 M=${F#maps/}
25                 M=${M%.map.options}
26                 blobhash=`git ls-files -s -- "$F" | cut -d ' ' -f 2`-`git ls-files -s -- "${F%.options}" | cut -d ' ' -f 2`
27                 "$@" "$M" "$blobhash"
28         done
29 }
30
31 pre2spam()
32 {
33         map=$1
34         url=$2
35         branch=$3
36         hash=$4
37
38         branch=${branch##refs/heads/}
39         branch=${branch##refs/remotes/}
40         branch=${branch##origin/}
41         hash=`echo "$hash" | cut -c 1-7`
42
43         echo "[$branch $hash] starting map compile of $map"
44 }
45
46 ss2spam()
47 {
48         map=$1
49         url=$2
50         branch=$3
51         hash=$4
52         status=$5
53
54         branch=${branch##refs/heads/}
55         branch=${branch##refs/remotes/}
56         branch=${branch##origin/}
57         hash=`echo "$hash" | cut -c 1-7`
58
59         s_error=
60
61         if [ "$status" -ne 0 ]; then
62                 s_error="exited with status $status"
63         fi
64
65         if [ -n "$s_error" ]; then
66                 echo "[$branch $hash] \ 34screenshots of $map could not be made: $s_error"
67         else
68                 echo "[$branch $hash] screenshots of $map are available on $url"
69         fi
70 }
71
72 log2spam()
73 {
74         map=$1
75         url=$2
76         branch=$3
77         hash=$4
78         status=$5
79         time=$6
80
81         hash=`echo "$hash" | cut -c 1-7`
82
83         branch=${branch##refs/heads/}
84         branch=${branch##refs/remotes/}
85         branch=${branch##origin/}
86
87         s_samplesize=
88         s_failshaders=
89         s_leaked=
90         s_error=
91
92         if [ "$status" -ne 0 ]; then
93                 s_error="exited with status $status"
94         fi
95         while IFS= read -r L; do
96                 case "$L" in
97                         WARNING:\ surface\ at\ *\ too\ large\ for\ desired\ samplesize*)
98                                 s=${L##* }
99                                 if [ -z "$s_samplesize" ] || [ "$s" -gt "$s_samplesize" ]; then
100                                         s_samplesize=$s
101                                 fi
102                                 ;;
103                         WARNING:\ Couldn\'t\ find\ image\ for\ shader\ textures/NULL)
104                                 # radiant stupid
105                                 ;;
106                         WARNING:\ Couldn\'t\ find\ image\ for\ shader\ *)
107                                 s_failshaders="$s_failshaders ${L##* }"
108                                 ;;
109                         '******* leaked *******')
110                                 s_leaked=1
111                                 ;;
112                         '************ ERROR ************')
113                                 IFS= read -r s_error
114                                 ;;
115                 esac
116         done
117         s_failshaders=`echo "$s_failshaders" | sed "s, textures/, ,g"`
118         s_failshaders=${s_failshaders# }
119         if [ -n "$s_error" ]; then
120                 echo -n "[$branch $hash] \ 34failed"
121         else
122                 echo -n "[$branch $hash] finished"
123         fi
124         echo -n " map compile of $map ($url): $time sec"
125         if [ -n "$s_samplesize" ]; then
126                 echo -n ", FIX samplesize >= $s_samplesize"
127         fi
128         if [ -n "$s_failshaders" ]; then
129                 if [ -n "`echo "$s_failshaders" | cut -d ' ' -f 4-`" ]; then
130                         s_failshaders="`echo "$s_failshaders" | cut -d ' ' -f 1-3`..."
131                 fi
132                 echo -n ", FIX shaders $s_failshaders"
133         fi
134         if [ -n "$s_leaked" ]; then
135                 echo -n ", FIX LEAK"
136         fi
137         if [ -n "$s_error" ]; then
138                 s_error=`echo "$s_error" | sed "s,$PWD/\?,,g"`
139                 echo -n ", ERROR: $s_error"
140         fi
141         echo
142 }
143
144 buildthemap()
145 {
146         REFNAME=$1
147         HASH=$2
148         url=$3
149         M=$4
150         blobhash=$5
151         if HEAD "$url$M-$blobhash.pk3"; then
152                 continue
153         fi
154         if [ -n "$IRCSPAM" ]; then
155                 pre2spam "$M" "$url$M-$blobhash.pk3" "$REFNAME" "$HASH" | $IRCSPAM
156         fi
157         t0=`date +%s`
158         (
159                 cd maps
160                 ../../../misc/tools/xonotic-map-compiler "$M" `grep ^- "$M.map.options" | cut -d '#' -f 1` $build_override > "$M.log"
161         )
162         t1=`date +%s`
163         dt=$(($t1 - $t0))
164         status=$?
165         if [ -n "$IRCSPAM" ]; then
166                 cat "maps/$M.log" | log2spam "$M" "$url$M-$blobhash.pk3" "$REFNAME" "$HASH" "$status" "$dt" > "maps/$M.irc"
167         fi
168         zip -9r "$M-$blobhash.pk3" "maps/$M.bsp" "maps/$M.log" "maps/$M.irc" "maps/$M/" "maps/$M.lin" "gfx/${M}_mini.tga"
169         ln -snf "../$M-$blobhash.pk3" "$M.pk3" # from ALL branches, so beware!
170         cp "$M-$blobhash.pk3" "$M-full-$blobhash.pk3"
171         zip -9r "$M-full-$blobhash.pk3" `git diff --name-only --diff-filter=ACMRTUXB master...HEAD` || true
172         ln -snf "../$M-full-$blobhash.pk3" "$M-full.pk3" # from ALL branches, so beware!
173         rsync -vaSHP "$M-$blobhash.pk3" "$M-full-$blobhash.pk3" "$url_ssh"
174         rsync -vaSHP "$M.pk3" "$M-full.pk3" "$url_ssh""latest/"
175         if [ -n "$IRCSPAM" ]; then
176                 $IRCSPAM < "maps/$M.irc"
177         fi
178 }
179
180 screenshotthemap()
181 {
182         REFNAME=$1
183         HASH=$2
184         url=$3
185         M=$4
186         blobhash=$5
187         if HEAD "$url$M-$blobhash/"; then
188                 continue
189         fi
190         rm -rf ~/.xonotic
191         (
192                 cd ../..
193                 if [ -n "$DISPLAY" ]; then
194                         misc/tools/xonotic-map-screenshot "$M" $screenshot_override +"scr_screenshot_name \"$M-\""
195                 else
196                         startx "$PWD/misc/tools/xonotic-map-screenshot" "$M" $screenshot_override +"scr_screenshot_name \"$M-\"" -- :8
197                 fi
198         )
199         if ! mv ~/.xonotic/data/screenshots "$M-$blobhash"; then
200                 if [ -n "$IRCSPAM" ]; then
201                         ss2spam "$M" "$url$M-$blobhash/" "$REFNAME" "$HASH" 1 > "maps/$M.ircss"
202                 fi
203                 return 1
204         fi
205         if [ -n "$IRCSPAM" ]; then
206                 ss2spam "$M" "$url$M-$blobhash/" "$REFNAME" "$HASH" 0 > "maps/$M.ircss"
207         fi
208         ln -snf "../$M-$blobhash" "$M" # from ALL branches, so beware!
209         rsync -vaSHP "$M-$blobhash" "$url_ssh"
210         rsync -vaSHP "$M" "$url_ssh""latest/"
211         if [ -n "$IRCSPAM" ]; then
212                 $IRCSPAM < "maps/$M.ircss"
213         fi
214 }
215
216 getthemap()
217 {
218         url=$1
219         bspdir_old=$2
220         bspdir=$3
221         M=$4
222         blobhash=$5
223         if mv "$bspdir_old/$M-$blobhash.pk3" "$bspdir/$M-$blobhash.pk3"; then
224                 continue
225         fi
226         if ! wget -O "$bspdir/$M-$blobhash.pk3" "$url$M-$blobhash.pk3"; then
227                 if ! curl -o "$bspdir/$M-$blobhash.pk3" "$url$M-$blobhash.pk3"; then
228                         rm -f "$bspdir/$M-$blobhash.pk3"
229                         echo "WARNING: could not download $url$M-$blobhash.pk3, maybe not ready yet"
230                         getthemap_fail=true
231                         return 0
232                 fi
233         fi
234         if ! unzip -l "$bspdir/$M-$blobhash.pk3"; then
235                 rm -f "$bspdir/$M-$blobhash.pk3"
236                 echo "WARNING: could not download $url$M-$blobhash.pk3, invalid zip file"
237                 getthemap_fail=true
238                 return 0
239         fi
240 }
241
242 case "$1" in
243         build)
244                 cd data/xonotic-maps.pk3dir
245                 git for-each-ref 'refs/remotes' | while read -r HASH TYPE REFNAME; do
246                         if [ -f "$build_cachedir/$HASH" ]; then
247                                 continue
248                         fi
249                         git reset --hard
250                         git clean -xfd
251                         git checkout -f "$HASH"
252                         allmaps buildthemap "$REFNAME" "$HASH" "$http_url"
253                         touch "$build_cachedir/$HASH"
254                 done
255                 git checkout -f master
256                 ;;
257         screenshot)
258                 cd data/xonotic-maps.pk3dir
259                 git for-each-ref 'refs/remotes' | while read -r HASH TYPE REFNAME; do
260                         if [ -f "$screenshot_cachedir/$HASH" ]; then
261                                 continue
262                         fi
263                         git reset --hard
264                         git clean -xfd
265                         git checkout -f "$HASH"
266
267                         cd ../..
268                         mkdir -p "$bspdir" "$bspdir.old"
269                         for b in "$bspdir"/*-????????????????????????????????????????-????????????????????????????????????????.pk3; do
270                                 if [ -e "$b" ]; then
271                                         mv "$b" "$bspdir.old"/
272                                 fi
273                         done
274                         cd data/xonotic-maps.pk3dir
275                         allmaps getthemap "$http_url" "$bspdir.old" "$bspdir"
276
277                         if ! $getthemap_fail; then
278                                 allmaps screenshotthemap "$REFNAME" "$HASH" "$http_url"
279                                 touch "$screenshot_cachedir/$HASH"
280                         fi
281                 done
282                 git checkout -f master
283                 ;;
284         download)
285                 mkdir -p "$bspdir" "$bspdir.old"
286                 for b in "$bspdir"/*-????????????????????????????????????????-????????????????????????????????????????.pk3; do
287                         if [ -e "$b" ]; then
288                                 mv "$b" "$bspdir.old"/
289                         fi
290                 done
291                 cd data/xonotic-maps.pk3dir
292                 allmaps getthemap "$http_url" "$bspdir.old" "$bspdir"
293                 echo "List of maps that got deleted (if any) and currently are in $bspdir.old:"
294                 ls -l "$bspdir.old"
295                 ;;
296         download-latest)
297                 mkdir -p "$bspdir"
298                 cd "$bspdir"
299                 rm -f *-????????????????????????????????????????-????????????????????????????????????????.pk3
300                 wget -r -l1 -A "*.pk3" -N --no-parent --no-directories "$http_url""latest"
301                 ;;
302         log2spam-test)
303                 log2spam "mapname" "http://mapurl" "branch" "commit" "0"
304                 ;;
305 esac