Merge branch 'cmake-q3map2' into 'master'
[xonotic/xonotic.git] / misc / tools / cached-converter.sh
1 #!/bin/sh
2
3 set -e
4
5 : ${CACHEDIR:=$HOME/.xonotic-cached-converter}
6 : ${do_jpeg:=true}
7 : ${do_jpeg_if_not_dds:=false}
8 : ${jpeg_qual_rgb:=95}
9 : ${jpeg_qual_a:=99}
10 : ${do_dds:=true}
11 : ${dds_tool:=compressonator-dxtc}
12 : ${do_ogg:=false}
13 : ${ogg_ogg:=true}
14 : ${ogg_qual:=2}
15 : ${del_src:=false}
16 : ${git_src_repo:=}
17 : ${dds_noalpha:=dxt1}
18 : ${dds_prealpha:=dxt2 dxt4}
19 : ${dds_sepalpha:=dxt3 dxt5}
20
21 selfprofile_t0=`date +%s`
22 selfprofile_step=init
23 selfprofile()
24 {
25         selfprofile_t=`date +%s`
26         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step+$selfprofile_t))"
27         selfprofile_step=$1
28         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step-$selfprofile_t))"
29         selfprofile_t0=$selfprofile_t
30 }
31
32 me=$0
33 case "$me" in
34         */*)
35                 meprefix=${me%/*}/
36                 ;;
37         *)
38                 meprefix=
39                 ;;
40 esac
41
42 tmpdir=`mktemp -d -t cached-converter.XXXXXX`
43 trap 'exit 1' INT
44 trap 'rm -rf "$tmpdir"' EXIT
45
46
47 use_magnet_to_acquire_checksum_faster()
48 #             ___________________
49 #        ,--'' ~~~~~~~^^^~._     '.
50 #    ,.-' ~~~~~~~~~~^^^^^~~~._._   \
51 #    |   /^^^^^|    /^^^^^^^^\\ \   \
52 #  ,/___  <  o>      <  (OO) > _     \
53 # /'/,         |-         .       ----.\
54 # |(|-'^^;,-  ,|     __    ^~~^^^^^^^; |\
55 # \\`  |    <;_    __ |`---  ..-^^/- | ||
56 #  \`-|Oq-.____`________~='^^|__,/  ' //
57 #   \ || | |   |  |    \ ..-;|  /    '/
58 #   | ||#|#|the|==|game!|'^` |/'    /'
59 #   | \\\\^\***|***|    \ ,,;'     /
60 #   |  `-=\_\__\___\__..-' ,.- - ,/
61 #   | . `-_  ------   _,-'^-'^,-'
62 #   | `-._________..--''^,-''^
63 #   \             ,...-'^
64 #    `----------'^              PROBLEM?
65 {
66         magnet=`GIT_DIR="$git_src_repo/.git" git ls-files -s "$1"`
67         if [ -n "$magnet" ]; then
68                 magnet=${magnet#* }
69                 magnet=${magnet%% *}
70                 echo "$magnet"
71         else
72                 git hash-object "$1"
73         fi
74 }
75
76 lastinfiles=
77 lastinfileshash=
78 acquire_checksum()
79 {
80         if [ x"$1/../$2" = x"$lastinfiles" ]; then
81                 _a_s=$lastinfileshash
82         else
83                 _a_e=false
84                 for _a_f in "$1" "$2"; do
85                         case "$_a_f" in
86                                 */background_l2.tga|*/background_ingame_l2.tga)
87                                         _a_e=true
88                                         ;;
89                         esac
90                 done
91                 if [ -n "$git_src_repo" ] && ! $_a_e; then
92                         _a_s=`use_magnet_to_acquire_checksum_faster "${1#./}"`
93                         if [ -n "$2" ]; then
94                                 _a_s=$_a_s`use_magnet_to_acquire_checksum_faster "${2#./}"`
95                         fi
96                 else
97                         _a_s=`git hash-object "$1"`
98                         if [ -n "$2" ]; then
99                                 _a_s=$_a_s`git hash-object "$2"`
100                         fi
101                 fi
102                 lastinfileshash=$_a_s
103                 lastinfiles="$1/../$2"
104         fi
105         echo "$_a_s"
106 }
107
108 cached()
109 {
110         flag=$1; shift
111         method=$1; shift
112         infile1=$1; shift
113         infile2=$1; shift
114         outfile1=$1; shift
115         outfile2=$1; shift
116         if ! $flag; then
117                 return 0
118         fi
119         #sleep 0.25
120         if [ x"$infile1" = x"$outfile1" ]; then
121                 keep=true
122         fi
123         options=`echo "$*" | git hash-object --stdin`
124         selfprofile convert_findchecksum
125         sum=`acquire_checksum "$infile1" "$infile2"`
126         selfprofile convert_makecachedir
127         mkdir -p "$CACHEDIR/$method-$options"
128         name1="$CACHEDIR/$method-$options/$sum-1.${outfile1##*.}"
129         [ -z "$outfile2" ] || name2="$CACHEDIR/$method-$options/$sum-2.${outfile2##*.}"
130         tempfile1="${name1%/*}/new-${name1##*/}"
131         [ -z "$outfile2" ] || tempfile2="${name2%/*}/new-${name2##*/}"
132         if [ -f "$name1" ] && { [ -z "$outfile2" ] || [ -f "$name2" ]; }; then
133                 selfprofile convert_copyoutput
134                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
135                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
136                 conv=true
137         elif selfprofile convert_makeoutput; "$method" "$infile1" "$infile2" "$tempfile1" "$tempfile2" "$@"; then
138                 mv "$tempfile1" "$name1"
139                 [ -z "$outfile2" ] || mv "$tempfile2" "$name2"
140                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
141                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
142                 conv=true
143         else
144                 selfprofile convert_cleartemp
145                 rm -f "$tempfile1"
146                 rm -f "$tempfile2"
147                 selfprofile convert_finished
148                 exit 1
149         fi
150         selfprofile convert_finished
151 }
152
153 pickdxta()
154 {
155         pd_t=$1; shift
156         pd_d=$1; shift
157         pd_i=$1; shift
158         pd_o=$1; shift
159         for pd_dd in $pd_d; do
160                 if [ -f "$pd_o" ]; then
161                         "$meprefix"compress-texture "$pd_t" "$pd_dd" "$pd_i" "$pd_o".tmp.dds "$@"
162                         pd_psnr_tmp=`compare -channel alpha -metric PSNR "$pd_i" "$pd_o".tmp.dds NULL: 2>&1`
163                         case "$pd_psnr_tmp" in
164                                 [0-9.]*)
165                                         ;;
166                                 *)
167                                         pd_psnr_tmp=999.9
168                                         ;;
169                         esac
170                         echo >&2 "$pd_dd: $pd_psnr_tmp dB"
171                         pd_psnr_diff=`echo "($pd_psnr_tmp) - ($pd_psnr)" | bc -l`
172                         case "$pd_psnr_diff" in
173                                 -*|0)
174                                         # tmp is smaller or equal
175                                         # smaller PSNR is worse
176                                         # no action
177                                         ;;
178                                 *)
179                                         # tmp is larger
180                                         # larger PSNR is better
181                                         pd_psnr=$pd_psnr_tmp
182                                         mv "$pd_o".tmp.dds "$pd_o"
183                                         echo >&2 "PICKED (better)"
184                                         ;;
185                         esac
186                 else
187                         "$meprefix"compress-texture "$pd_t" "$pd_dd" "$pd_i" "$pd_o" "$@"
188                         pd_psnr=`compare -channel alpha -metric PSNR "$pd_i" "$pd_o" NULL: 2>&1`
189                         case "$pd_psnr" in
190                                 [0-9.]*)
191                                         ;;
192                                 *)
193                                         pd_psnr=999.9
194                                         ;;
195                         esac
196                         echo >&2 "$pd_dd: $pd_psnr dB"
197                         echo >&2 "PICKED (first)"
198                 fi
199         done
200 }
201
202 reduce_jpeg2_dds()
203 {
204         i=$1; shift
205         ia=$1; shift
206         o=$1; shift; shift 
207         convert "$i" "$ia" -compose CopyOpacity -composite -type TrueColorMatte "$tmpdir/x.tga" && \
208         pickdxta "$dds_tool" "$dds_sepalpha" "$tmpdir/x.tga" "$o" $1
209 }
210
211 reduce_jpeg2_dds_premul()
212 {
213         i=$1; shift
214         ia=$1; shift
215         o=$1; shift; shift 
216         convert "$i" "$ia" -compose CopyOpacity -composite -type TrueColorMatte "$tmpdir/x.tga" && \
217         pickdxta "$dds_tool" "$dds_prealpha" "$tmpdir/x.tga" "$o" $1
218 }
219
220 reduce_jpeg2_jpeg2()
221 {
222         i=$1; shift
223         ia=$1; shift
224         o=$1; shift
225         oa=$1; shift
226         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
227                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
228                         cp "$i" "$o"
229                 fi
230         else
231                 return 1
232         fi
233         if convert "$ia" TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
234                 if [ "`stat -c %s "$ia"`" -lt "`stat -c %s "$oa"`" ]; then
235                         cp "$ia" "$oa"
236                 fi
237         else
238                 return 1
239         fi
240 }
241
242 reduce_jpeg_jpeg()
243 {
244         i=$1; shift; shift
245         o=$1; shift; shift
246         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
247                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
248                         cp "$i" "$o"
249                 fi
250         else
251                 return 1
252         fi
253 }
254
255 reduce_ogg_ogg()
256 {
257         i=$1; shift; shift
258         o=$1; shift; shift
259         tags=`vorbiscomment -R -l "$i" || true`
260         oggdec -o "$tmpdir/x.wav" "$i" && \
261         oggenc -q"$1" -o "$o" "$tmpdir/x.wav"
262         echo "$tags" | vorbiscomment -R -w "$o" || true
263 }
264
265 reduce_wav_ogg()
266 {
267         i=$1; shift; shift
268         o=$1; shift; shift
269         oggenc -q"$1" -o "$o" "$i"
270 }
271
272 reduce_rgba_dds()
273 {
274         i=$1; shift; shift
275         o=$1; shift; shift
276         convert "$i" -type TrueColorMatte "$tmpdir/x.tga" && \
277         pickdxta "$dds_tool" "$dds_sepalpha" "$tmpdir/x.tga" "$o" $1
278 }
279
280 reduce_rgba_dds_premul()
281 {
282         i=$1; shift; shift
283         o=$1; shift; shift
284         convert "$i" -type TrueColorMatte "$tmpdir/x.tga" && \
285         pickdxta "$dds_tool" "$dds_prealpha" "$tmpdir/x.tga" "$o" $1
286 }
287
288 reduce_rgba_jpeg2()
289 {
290         i=$1; shift; shift
291         o=$1; shift
292         oa=$1; shift
293         if convert "$i" -alpha off TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
294                 :
295         else
296                 return 1
297         fi
298         if convert "$i" -alpha extract TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
299                 :
300         else
301                 return 1
302         fi
303 }
304
305 reduce_rgb_dds()
306 {
307         i=$1; shift; shift
308         o=$1; shift; shift
309         convert "$i" -type TrueColor "$tmpdir/x.tga" && \
310         "$meprefix"compress-texture "$dds_tool" "$dds_noalpha" "$tmpdir/x.tga" "$o" $1
311 }
312
313 reduce_rgb_jpeg()
314 {
315         i=$1; shift; shift
316         o=$1; shift; shift
317         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
318                 :
319         else
320                 return 1
321         fi
322 }
323
324 has_alpha()
325 {
326         i=$1; shift; shift
327         o=$1; shift; shift
328         if convert "$i" -depth 16 RGBA:- | perl -e 'while(read STDIN, $_, 8) { substr($_, 6, 2) eq "\xFF\xFF" or exit 1; } exit 0;'; then
329                 # no alpha
330                 : > "$o"
331         else
332                 # has alpha
333                 echo yes > "$o"
334         fi
335 }
336
337 to_delete=
338 for F in "$@"; do
339         selfprofile prepareconvert
340         f=${F%.*}
341
342         echo >&2 "Handling $F..."
343         conv=false
344         keep=false
345         jqual_rgb=$jpeg_qual_rgb
346         jqual_a=$jpeg_qual_a
347
348         will_jpeg=$do_jpeg
349         will_dds=$do_dds
350         will_ogg=$do_ogg
351         if ! $ogg_ogg; then
352                 case "$f" in
353                         *.ogg) will_ogg=false ;;
354                 esac
355         fi
356         case "$f" in
357                 ./sounds/misc/talk*.wav) will_ogg=false ;; # engine "feature"
358                 *_bump) will_dds=false ;;
359                 ./models/player/*) will_dds=false ;;
360                 ./models/sprites/*) will_dds=false ;;
361                 ./textures/*) ;;
362                 ./models/*) ;;
363                 ./particles/*) ;;
364                 ./progs/*) ;;
365                 *)
366                         # we can't DDS compress the 2D textures, sorry
367                         # but JPEG is still fine
368                         will_dds=false
369                         ;;
370         esac
371
372         # configure S2TC
373         case "$f" in
374                 *_norm)
375                         export S2TC_COLORDIST_MODE=NORMALMAP
376                         export S2TC_RANDOM_COLORS=256
377                         export S2TC_REFINE_COLORS=LOOP
378                         ;;
379                 *)
380                         export S2TC_COLORDIST_MODE=SRGB_MIXED
381                         export S2TC_RANDOM_COLORS=64
382                         export S2TC_REFINE_COLORS=LOOP
383                         ;;
384         esac
385
386         # for deluxemaps, lightmaps and normalmaps, enforce high jpeg quality (like on alpha channels)
387         if [ "$jqual_a" -gt "$jqual_rgb" ]; then
388                 case "$f" in
389                         ./maps/*/lm_[0-9][0-9][0-9][13579]) # deluxemap
390                                 jqual_rgb=$jqual_a
391                                 ;;
392                         ./maps/*/lm_[0-9][0-9][0-9][02468]) # lightmap
393                                 jqual_rgb=$jqual_a
394                                 ;;
395                         *_norm) # normalmap
396                                 jqual_rgb=$jqual_a
397                                 ;;
398                 esac
399         fi
400
401         pm=
402         case "$f" in
403                 ./particles/particlefont) # particlefont uses premultiplied alpha
404                         pm=_premul
405                         ;;
406         esac
407
408         if $do_jpeg_if_not_dds; then
409                 if $will_dds; then
410                         will_jpeg=false
411                 else
412                         will_jpeg=true
413                 fi
414         fi
415         selfprofile startconvert
416         case "$F" in
417                 *_alpha.jpg)
418                         # handle in *.jpg case
419
420                         # they always got converted, I assume
421                         if $will_dds || $will_jpeg; then
422                                 conv=true
423                         fi
424                         keep=$will_jpeg
425                         ;;
426                 *.jpg)
427                         if [ -f "${f}_alpha.jpg" ]; then
428                                 cached "$will_dds"  reduce_jpeg2_dds$pm "$F" "${f}_alpha.jpg" "dds/${f}.dds" ""               "$dds_flags"
429                                 cached "$will_jpeg" reduce_jpeg2_jpeg2  "$F" "${f}_alpha.jpg" "$F"           "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
430                         else                                   
431                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
432                                 cached "$will_jpeg" reduce_jpeg_jpeg    "$F" ""               "$F"           ""               "$jqual_rgb"
433                         fi
434                         ;;
435                 *.png|*.tga)
436                         cached true has_alpha "$F" "" "$F.hasalpha" ""
437                         conv=false
438                         if [ -s "$F.hasalpha" ]; then
439                                 cached "$will_dds"  reduce_rgba_dds$pm  "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
440                                 cached "$will_jpeg" reduce_rgba_jpeg2   "$F" ""               "${f}.jpg"     "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
441                         else                                                             
442                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
443                                 cached "$will_jpeg" reduce_rgb_jpeg     "$F" ""               "${f}.jpg"     ""               "$jqual_rgb"
444                         fi
445                         rm -f "$F.hasalpha"
446                         ;;
447                 *.ogg)
448                         cached "$will_ogg" reduce_ogg_ogg "$F" "" "$F" "" "$ogg_qual"
449                         ;;
450                 ./sound/misc/null.wav)
451                         # never convert this one
452                         ;;
453                 *.wav)
454                         cached "$will_ogg" reduce_wav_ogg "$F" "" "${f}.ogg" "" "$ogg_qual"
455                         ;;
456         esac
457         selfprofile marktodelete
458         if $del_src; then
459                 if $conv; then
460                         if ! $keep; then
461                                 # FIXME can't have spaces in filenames that way
462                                 to_delete="$to_delete $F"
463                         fi
464                 fi
465         fi
466         selfprofile symlinkfixing
467         # fix up DDS paths by a symbolic link
468         if [ -f "dds/${f}.dds" ]; then
469                 if [ -z "${f##./textures/*}" ]; then
470                         if [ -n "${f##./textures/*/*}" ]; then
471                                 ln -snf "textures/${f#./textures/}.dds" "dds/${f#./textures/}.dds"
472                         fi
473                 fi
474         fi
475         selfprofile looping
476 done
477
478 for F in $to_delete; do
479         rm -f "$F"
480 done
481 selfprofile finished_time
482 set | grep ^selfprofile_counter_ >&2