reban
[xonotic/xonotic.git] / misc / tools / cached-converter.sh
1 #!/bin/sh
2
3 set -e
4
5 : ${CACHEDIR:=$HOME/.xonotic-cached-converter}
6 : ${do_jpeg:=true}
7 : ${do_jpeg_if_not_dds:=false}
8 : ${jpeg_qual_rgb:=95}
9 : ${jpeg_qual_a:=99}
10 : ${do_webp:=false}
11 : ${do_webp_if_not_dds:=false}
12 : ${webp_flags_lq:=-lossless -q 100}
13 : ${webp_flags_hq:=-lossless -q 100}
14 : ${webp_flags_alq:=-lossless -q 100 -q_alpha 100}
15 : ${webp_flags_ahq:=-lossless -q 100 -q_alpha 100}
16 : ${do_dds:=true}
17 : ${dds_tool:=s2tc}
18 : ${do_ogg:=false}
19 : ${ogg_ogg:=true}
20 : ${ogg_qual:=2}
21 : ${del_src:=false}
22 : ${git_src_repo:=}
23 : ${dds_noalpha:=dxt1}
24 : ${dds_prealpha:=dxt2 dxt4}
25 : ${dds_sepalpha:=dxt3 dxt5}
26
27 selfprofile_t0=`date +%s`
28 selfprofile_step=init
29 selfprofile()
30 {
31         selfprofile_t=`date +%s`
32         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step+$selfprofile_t))"
33         selfprofile_step=$1
34         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step-$selfprofile_t))"
35         selfprofile_t0=$selfprofile_t
36 }
37
38 me=$0
39 case "$me" in
40         */*)
41                 meprefix=${me%/*}/
42                 ;;
43         *)
44                 meprefix=
45                 ;;
46 esac
47
48 tmpdir=`mktemp -d -t cached-converter.XXXXXX`
49 trap 'exit 1' INT
50 trap 'rm -rf "$tmpdir"' EXIT
51
52
53 use_magnet_to_acquire_checksum_faster()
54 #             ___________________
55 #        ,--'' ~~~~~~~^^^~._     '.
56 #    ,.-' ~~~~~~~~~~^^^^^~~~._._   \
57 #    |   /^^^^^|    /^^^^^^^^\\ \   \
58 #  ,/___  <  o>      <  (OO) > _     \
59 # /'/,         |-         .       ----.\
60 # |(|-'^^;,-  ,|     __    ^~~^^^^^^^; |\
61 # \\`  |    <;_    __ |`---  ..-^^/- | ||
62 #  \`-|Oq-.____`________~='^^|__,/  ' //
63 #   \ || | |   |  |    \ ..-;|  /    '/
64 #   | ||#|#|the|==|game!|'^` |/'    /'
65 #   | \\\\^\***|***|    \ ,,;'     /
66 #   |  `-=\_\__\___\__..-' ,.- - ,/
67 #   | . `-_  ------   _,-'^-'^,-'
68 #   | `-._________..--''^,-''^
69 #   \             ,...-'^
70 #    `----------'^              PROBLEM?
71 {
72         magnet=`GIT_DIR="$git_src_repo/.git" git ls-files -s "$1"`
73         if [ -n "$magnet" ]; then
74                 magnet=${magnet#* }
75                 magnet=${magnet%% *}
76                 echo "$magnet"
77         else
78                 git hash-object "$1"
79         fi
80 }
81
82 lastinfiles=
83 lastinfileshash=
84 acquire_checksum()
85 {
86         if [ x"$1/../$2" = x"$lastinfiles" ]; then
87                 _a_s=$lastinfileshash
88         else
89                 _a_e=false
90                 for _a_f in "$1" "$2"; do
91                         case "$_a_f" in
92                                 */background_l2.tga|*/background_ingame_l2.tga)
93                                         _a_e=true
94                                         ;;
95                         esac
96                 done
97                 if [ -n "$git_src_repo" ] && ! $_a_e; then
98                         _a_s=`use_magnet_to_acquire_checksum_faster "${1#./}"`
99                         if [ -n "$2" ]; then
100                                 _a_s=$_a_s`use_magnet_to_acquire_checksum_faster "${2#./}"`
101                         fi
102                 else
103                         _a_s=`git hash-object "$1"`
104                         if [ -n "$2" ]; then
105                                 _a_s=$_a_s`git hash-object "$2"`
106                         fi
107                 fi
108                 lastinfileshash=$_a_s
109                 lastinfiles="$1/../$2"
110         fi
111         echo "$_a_s"
112 }
113
114 cached()
115 {
116         flag=$1; shift
117         method=$1; shift
118         infile1=$1; shift
119         infile2=$1; shift
120         outfile1=$1; shift
121         outfile2=$1; shift
122         if ! $flag; then
123                 return 0
124         fi
125         #sleep 0.25
126         if [ x"$infile1" = x"$outfile1" ]; then
127                 keep=true
128         fi
129         options=`echo "$*" | git hash-object --stdin`
130         selfprofile convert_findchecksum
131         sum=`acquire_checksum "$infile1" "$infile2"`
132         selfprofile convert_makecachedir
133         mkdir -p "$CACHEDIR/$method-$options"
134         name1="$CACHEDIR/$method-$options/$sum-1.${outfile1##*.}"
135         [ -z "$outfile2" ] || name2="$CACHEDIR/$method-$options/$sum-2.${outfile2##*.}"
136         tempfile1="${name1%/*}/new-${name1##*/}"
137         [ -z "$outfile2" ] || tempfile2="${name2%/*}/new-${name2##*/}"
138         if [ -f "$name1" ] && { [ -z "$outfile2" ] || [ -f "$name2" ]; }; then
139                 selfprofile convert_copyoutput
140                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
141                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
142                 conv=true
143         elif selfprofile convert_makeoutput; "$method" "$infile1" "$infile2" "$tempfile1" "$tempfile2" "$@"; then
144                 mv "$tempfile1" "$name1"
145                 [ -z "$outfile2" ] || mv "$tempfile2" "$name2"
146                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
147                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
148                 conv=true
149         else
150                 selfprofile convert_cleartemp
151                 rm -f "$tempfile1"
152                 rm -f "$tempfile2"
153                 selfprofile convert_finished
154                 exit 1
155         fi
156         selfprofile convert_finished
157 }
158
159 pickdxta()
160 {
161         pd_t=$1; shift
162         pd_d=$1; shift
163         pd_i=$1; shift
164         pd_o=$1; shift
165         for pd_dd in $pd_d; do
166                 if [ -f "$pd_o" ]; then
167                         "$meprefix"compress-texture "$pd_t" "$pd_dd" "$pd_i" "$pd_o".tmp.dds "$@"
168                         pd_psnr_tmp=`compare -channel alpha -metric PSNR "$pd_i" "$pd_o".tmp.dds NULL: 2>&1`
169                         case "$pd_psnr_tmp" in
170                                 [0-9.]*)
171                                         ;;
172                                 *)
173                                         pd_psnr_tmp=999.9
174                                         ;;
175                         esac
176                         echo >&2 "$pd_dd: $pd_psnr_tmp dB"
177                         pd_psnr_diff=`echo "($pd_psnr_tmp) - ($pd_psnr)" | bc -l`
178                         case "$pd_psnr_diff" in
179                                 -*|0)
180                                         # tmp is smaller or equal
181                                         # smaller PSNR is worse
182                                         # no action
183                                         ;;
184                                 *)
185                                         # tmp is larger
186                                         # larger PSNR is better
187                                         pd_psnr=$pd_psnr_tmp
188                                         mv "$pd_o".tmp.dds "$pd_o"
189                                         echo >&2 "PICKED (better)"
190                                         ;;
191                         esac
192                 else
193                         "$meprefix"compress-texture "$pd_t" "$pd_dd" "$pd_i" "$pd_o" "$@"
194                         pd_psnr=`compare -channel alpha -metric PSNR "$pd_i" "$pd_o" NULL: 2>&1`
195                         case "$pd_psnr" in
196                                 [0-9.]*)
197                                         ;;
198                                 *)
199                                         pd_psnr=999.9
200                                         ;;
201                         esac
202                         echo >&2 "$pd_dd: $pd_psnr dB"
203                         echo >&2 "PICKED (first)"
204                 fi
205         done
206 }
207
208 reduce_jpeg2_dds()
209 {
210         i=$1; shift
211         ia=$1; shift
212         o=$1; shift; shift 
213         convert "$i" "$ia" -compose CopyOpacity -composite -type TrueColorMatte "$tmpdir/x.tga" && \
214         pickdxta "$dds_tool" "$dds_sepalpha" "$tmpdir/x.tga" "$o" $1
215 }
216
217 reduce_jpeg2_dds_premul()
218 {
219         i=$1; shift
220         ia=$1; shift
221         o=$1; shift; shift 
222         convert "$i" "$ia" -compose CopyOpacity -composite -type TrueColorMatte "$tmpdir/x.tga" && \
223         pickdxta "$dds_tool" "$dds_prealpha" "$tmpdir/x.tga" "$o" $1
224 }
225
226 reduce_jpeg2_jpeg2()
227 {
228         i=$1; shift
229         ia=$1; shift
230         o=$1; shift
231         oa=$1; shift
232         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
233                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
234                         cp "$i" "$o"
235                 fi
236         else
237                 return 1
238         fi
239         if convert "$ia" TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
240                 if [ "`stat -c %s "$ia"`" -lt "`stat -c %s "$oa"`" ]; then
241                         cp "$ia" "$oa"
242                 fi
243         else
244                 return 1
245         fi
246 }
247
248 reduce_jpeg2_webp()
249 {
250         i=$1; shift
251         ia=$1; shift
252         o=$1; shift; shift
253         # this one MUST run
254         convert "$i" "$ia" -compose CopyOpacity -composite -type TrueColorMatte "$tmpdir/x.png" && \
255         cwebp $1 "$tmpdir/x.png" -o "$o"
256 }
257
258 reduce_jpeg_jpeg()
259 {
260         i=$1; shift; shift
261         o=$1; shift; shift
262         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
263                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
264                         cp "$i" "$o"
265                 fi
266         else
267                 return 1
268         fi
269 }
270
271 reduce_ogg_ogg()
272 {
273         i=$1; shift; shift
274         o=$1; shift; shift
275         tags=`vorbiscomment -R -l "$i" || true`
276         oggdec -o "$tmpdir/x.wav" "$i" && \
277         oggenc -q"$1" -o "$o" "$tmpdir/x.wav"
278         echo "$tags" | vorbiscomment -R -w "$o" || true
279 }
280
281 reduce_wav_ogg()
282 {
283         i=$1; shift; shift
284         o=$1; shift; shift
285         oggenc -q"$1" -o "$o" "$i"
286 }
287
288 reduce_rgba_dds()
289 {
290         i=$1; shift; shift
291         o=$1; shift; shift
292         convert "$i" -type TrueColorMatte "$tmpdir/x.tga" && \
293         pickdxta "$dds_tool" "$dds_sepalpha" "$tmpdir/x.tga" "$o" $1
294 }
295
296 reduce_rgba_dds_premul()
297 {
298         i=$1; shift; shift
299         o=$1; shift; shift
300         convert "$i" -type TrueColorMatte "$tmpdir/x.tga" && \
301         pickdxta "$dds_tool" "$dds_prealpha" "$tmpdir/x.tga" "$o" $1
302 }
303
304 reduce_rgba_jpeg2()
305 {
306         i=$1; shift; shift
307         o=$1; shift
308         oa=$1; shift
309         if convert "$i" -alpha off TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
310                 :
311         else
312                 return 1
313         fi
314         if convert "$i" -alpha extract TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
315                 :
316         else
317                 return 1
318         fi
319 }
320
321 reduce_rgb_dds()
322 {
323         i=$1; shift; shift
324         o=$1; shift; shift
325         convert "$i" -type TrueColor "$tmpdir/x.tga" && \
326         "$meprefix"compress-texture "$dds_tool" "$dds_noalpha" "$tmpdir/x.tga" "$o" $1
327 }
328
329 reduce_rgb_jpeg()
330 {
331         i=$1; shift; shift
332         o=$1; shift; shift
333         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
334                 :
335         else
336                 return 1
337         fi
338 }
339
340 reduce_rgba_webp()
341 {
342         i=$1; shift; shift
343         o=$1; shift; shift
344         convert "$i" "$tmpdir/x.png" && \
345         cwebp $1 "$tmpdir/x.png" -o "$o"
346 }
347
348 has_alpha()
349 {
350         i=$1; shift; shift
351         o=$1; shift; shift
352         if convert "$i" -depth 16 RGBA:- | perl -e 'while(read STDIN, $_, 8) { substr($_, 6, 2) eq "\xFF\xFF" or exit 1; } exit 0;'; then
353                 # no alpha
354                 : > "$o"
355         else
356                 # has alpha
357                 echo yes > "$o"
358         fi
359 }
360
361 to_delete=
362 for F in "$@"; do
363         selfprofile prepareconvert
364         f=${F%.*}
365
366         echo >&2 "Handling $F..."
367         conv=false
368         keep=false
369         jqual_rgb=$jpeg_qual_rgb
370         jqual_a=$jpeg_qual_a
371         webp_mode=lq
372
373         will_jpeg=$do_jpeg
374         will_webp=$do_webp
375         will_dds=$do_dds
376         will_ogg=$do_ogg
377         if ! $ogg_ogg; then
378                 case "$F" in
379                         *.ogg) will_ogg=false ;;
380                 esac
381         fi
382         case "$F" in
383                 ./sound/misc/talk*.wav) will_ogg=false ;; # engine "feature"
384                 *_bump.*) will_dds=false ;;
385                 ./models/player/*) will_dds=false ;;
386                 ./models/sprites/*) will_dds=false ;;
387                 ./models/*) ;;
388                 ./textures/*) ;;
389                 ./models/*) ;;
390                 ./particles/*) ;;
391                 ./progs/*) ;;
392                 *)
393                         # we can't DDS compress the 2D textures, sorry
394                         # but JPEG is still fine
395                         will_dds=false
396                         ;;
397         esac
398
399         # configure S2TC
400         case "$f" in
401                 *_norm)
402                         export S2TC_COLORDIST_MODE=NORMALMAP
403                         export S2TC_RANDOM_COLORS=256
404                         export S2TC_REFINE_COLORS=LOOP
405                         ;;
406                 *)
407                         export S2TC_COLORDIST_MODE=SRGB_MIXED
408                         export S2TC_RANDOM_COLORS=64
409                         export S2TC_REFINE_COLORS=LOOP
410                         ;;
411         esac
412
413         # for deluxemaps, lightmaps and normalmaps, enforce high jpeg quality (like on alpha channels)
414         if [ "$jqual_a" -gt "$jqual_rgb" ]; then
415                 case "$f" in
416                         ./maps/*/lm_[0-9][0-9][0-9][13579]) # deluxemap
417                                 jqual_rgb=$jqual_a
418                                 webp_mode=hq
419                                 ;;
420                         ./maps/*/lm_[0-9][0-9][0-9][02468]) # lightmap
421                                 jqual_rgb=$jqual_a
422                                 webp_mode=hq
423                                 ;;
424                         *_norm) # normalmap
425                                 jqual_rgb=$jqual_a
426                                 webp_mode=hq
427                                 ;;
428                 esac
429         fi
430
431         pm=
432         case "$f" in
433                 ./particles/particlefont) # particlefont uses premultiplied alpha
434                         pm=_premul
435                         ;;
436         esac
437
438         if $do_jpeg_if_not_dds; then
439                 if $will_dds; then
440                         will_jpeg=false
441                 else
442                         will_jpeg=true
443                 fi
444         fi
445         if $do_webp_if_not_dds; then
446                 if $will_dds; then
447                         will_webp=false
448                 else
449                         will_webp=true
450                 fi
451         fi
452         selfprofile startconvert
453         case "$F" in
454                 *_alpha.jpg)
455                         # handle in *.jpg case
456
457                         # they always got converted, I assume
458                         if $will_dds || $will_jpeg || $will_webp; then
459                                 conv=true
460                         fi
461                         keep=$will_jpeg
462                         ;;
463                 *.jpg)
464                         if [ -f "${f}_alpha.jpg" ]; then
465                                 cached "$will_dds"  reduce_jpeg2_dds$pm "$F" "${f}_alpha.jpg" "dds/${f}.dds" ""               "$dds_flags"
466                                 cached "$will_jpeg" reduce_jpeg2_jpeg2  "$F" "${f}_alpha.jpg" "$F"           "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
467                                 #eval wflags=\$webp_flags_${webp_mode}a
468                                 #cached "$will_webp" reduce_jpeg2_webp   "$F" "${f}_alpha.jpg" "${f}.webp"    ""               "$wflags"
469                         else
470                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
471                                 cached "$will_jpeg" reduce_jpeg_jpeg    "$F" ""               "$F"           ""               "$jqual_rgb"
472                                 #eval wflags=\$webp_flags_${webp_mode}
473                                 #cached "$will_webp" reduce_rgba_webp    "$F" ""               "${f}.webp"    ""               "$wflags"
474                         fi
475                         ;;
476                 *.png|*.tga|*.webp)
477                         cached true has_alpha "$F" "" "$F.hasalpha" ""
478                         conv=false
479                         if [ -s "$F.hasalpha" ]; then
480                                 cached "$will_dds"  reduce_rgba_dds$pm  "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
481                                 cached "$will_jpeg" reduce_rgba_jpeg2   "$F" ""               "${f}.jpg"     "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
482                                 eval wflags=\$webp_flags_${webp_mode}a
483                                 cached "$will_webp" reduce_rgba_webp    "$F" ""               "${f}.webp"    ""               "$wflags"
484                         else
485                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
486                                 cached "$will_jpeg" reduce_rgb_jpeg     "$F" ""               "${f}.jpg"     ""               "$jqual_rgb"
487                                 eval wflags=\$webp_flags_${webp_mode}
488                                 cached "$will_webp" reduce_rgba_webp    "$F" ""               "${f}.webp"    ""               "$wflags"
489                         fi
490                         rm -f "$F.hasalpha"
491                         ;;
492                 *.ogg)
493                         cached "$will_ogg" reduce_ogg_ogg "$F" "" "$F" "" "$ogg_qual"
494                         ;;
495                 ./sound/misc/null.wav)
496                         # never convert this one
497                         ;;
498                 *.wav)
499                         cached "$will_ogg" reduce_wav_ogg "$F" "" "${f}.ogg" "" "$ogg_qual"
500                         ;;
501         esac
502         selfprofile marktodelete
503         if $del_src; then
504                 if $conv; then
505                         if ! $keep; then
506                                 # FIXME can't have spaces in filenames that way
507                                 to_delete="$to_delete $F"
508                         fi
509                 fi
510         fi
511         selfprofile symlinkfixing
512         # fix up DDS paths by a symbolic link
513         if [ -f "dds/${f}.dds" ]; then
514                 if [ -z "${f##./textures/*}" ]; then
515                         if [ -n "${f##./textures/*/*}" ]; then
516                                 ln -snf "textures/${f#./textures/}.dds" "dds/${f#./textures/}.dds"
517                         fi
518                 fi
519         fi
520         selfprofile looping
521 done
522
523 for F in $to_delete; do
524         rm -f "$F"
525 done
526 selfprofile finished_time
527 set | grep ^selfprofile_counter_ >&2