misc/tools/cached-converter.sh

   1 #!/bin/sh
   2
   3 set -e
   4
   5 : ${CACHEDIR:=$HOME/.xonotic-cached-converter}
   6 : ${do_jpeg:=true}
   7 : ${do_jpeg_if_not_dds:=false}
   8 : ${jpeg_qual_rgb:=95}
   9 : ${jpeg_qual_a:=99}
  10 : ${do_dds:=true}
  11 : ${dds_tool:=compressonator-dxtc}
  12 : ${do_ogg:=false}
  13 : ${ogg_qual:=1}
  14 : ${del_src:=false}
  15 : ${git_src_repo:=}
  16 : ${dds_noalpha:=dxt1}
  17 : ${dds_prealpha:=dxt2 dxt4}
  18 : ${dds_sepalpha:=dxt3 dxt5}
  19
  20 selfprofile_t0=`date +%s`
  21 selfprofile_step=init
  22 selfprofile()
  23 {
  24         selfprofile_t=`date +%s`
  25         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step+$selfprofile_t))"
  26         selfprofile_step=$1
  27         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step-$selfprofile_t))"
  28         selfprofile_t0=$selfprofile_t
  29 }
  30
  31 me=$0
  32 case "$me" in
  33         */*)
  34                 meprefix=${me%/*}/
  35                 ;;
  36         *)
  37                 meprefix=
  38                 ;;
  39 esac
  40
  41 tmpdir=`mktemp -d -t cached-converter.XXXXXX`
  42 trap 'exit 1' INT
  43 trap 'rm -rf "$tmpdir"' EXIT
  44
  45
  46 use_magnet_to_acquire_checksum_faster()
  47 #             ___________________
  48 #        ,--'' ~~~~~~~^^^~._     '.
  49 #    ,.-' ~~~~~~~~~~^^^^^~~~._._   \
  50 #    |   /^^^^^|    /^^^^^^^^\\ \   \
  51 #  ,/___  <  o>      <  (OO) > _     \
  52 # /'/,         |-         .       ----.\
  53 # |(|-'^^;,-  ,|     __    ^~~^^^^^^^; |\
  54 # \\`  |    <;_    __ |`---  ..-^^/- | ||
  55 #  \`-|Oq-.____`________~='^^|__,/  ' //
  56 #   \ || | |   |  |    \ ..-;|  /    '/
  57 #   | ||#|#|the|==|game!|'^` |/'    /'
  58 #   | \\\\^\***|***|    \ ,,;'     /
  59 #   |  `-=\_\__\___\__..-' ,.- - ,/
  60 #   | . `-_  ------   _,-'^-'^,-'
  61 #   | `-._________..--''^,-''^
  62 #   \             ,...-'^
  63 #    `----------'^              PROBLEM?
  64 {
  65         magnet=`GIT_DIR="$git_src_repo/.git" git ls-files -s "$1"`
  66         if [ -n "$magnet" ]; then
  67                 magnet=${magnet#* }
  68                 magnet=${magnet%% *}
  69                 sum=$sum$magnet
  70         else
  71                 sum=$sum`git hash-object "$1"`
  72         fi
  73 }
  74
  75 lastinfiles=
  76 lastinfileshash=
  77 cached()
  78 {
  79         flag=$1; shift
  80         method=$1; shift
  81         infile1=$1; shift
  82         infile2=$1; shift
  83         outfile1=$1; shift
  84         outfile2=$1; shift
  85         if ! $flag; then
  86                 return 0
  87         fi
  88         #sleep 0.25
  89         if [ x"$infile1" = x"$outfile1" ]; then
  90                 keep=true
  91         fi
  92         options=`echo "$*" | git hash-object --stdin`
  93         selfprofile convert_findchecksum
  94         if [ x"$infile1/../$infile2" = x"$lastinfiles" ]; then
  95                 sum=$lastinfileshash
  96         else
  97                 evil=false
  98                 for infile in "$infile1" "$infile2"; do
  99                         case "$infile" in
 100                                 */background_l2.tga|*/background_ingame_l2.tga)
 101                                         evil=true
 102                                         ;;
 103                         esac
 104                 done
 105                 if [ -n "$git_src_repo" ] && ! $evil; then
 106                         sum=
 107                         use_magnet_to_acquire_checksum_faster "${infile1#./}"
 108                         if [ -n "$infile2" ]; then
 109                                 use_magnet_to_acquire_checksum_faster "${infile2#./}"
 110                         fi
 111                 else
 112                         sum=`git hash-object "$infile1"`
 113                         if [ -n "$infile2" ]; then
 114                                 sum=$sum`git hash-object "$infile2"`
 115                         fi
 116                 fi
 117                 lastinfileshash=$sum
 118         fi
 119         selfprofile convert_makecachedir
 120         mkdir -p "$CACHEDIR/$method-$options"
 121         name1="$CACHEDIR/$method-$options/$sum-1.${outfile1##*.}"
 122         [ -z "$outfile2" ] || name2="$CACHEDIR/$method-$options/$sum-2.${outfile2##*.}"
 123         tempfile1="${name1%/*}/new-${name1##*/}"
 124         [ -z "$outfile2" ] || tempfile2="${name2%/*}/new-${name2##*/}"
 125         if [ -f "$name1" ] && { [ -z "$outfile2" ] || [ -f "$name2" ]; }; then
 126                 selfprofile convert_copyoutput
 127                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
 128                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
 129                 conv=true
 130         elif selfprofile convert_makeoutput; "$method" "$infile1" "$infile2" "$tempfile1" "$tempfile2" "$@"; then
 131                 mv "$tempfile1" "$name1"
 132                 [ -z "$outfile2" ] || mv "$tempfile2" "$name2"
 133                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
 134                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
 135                 conv=true
 136         else
 137                 selfprofile convert_cleartemp
 138                 rm -f "$tempfile1"
 139                 rm -f "$tempfile2"
 140                 selfprofile convert_finished
 141                 exit 1
 142         fi
 143         selfprofile convert_finished
 144 }
 145
 146 pickdxta()
 147 {
 148         pd_t=$1; shift
 149         pd_d=$1; shift
 150         pd_i=$1; shift
 151         pd_o=$1; shift
 152         for pd_dd in $pd_d; do
 153                 if [ -f "$pd_o".dds ]; then
 154                         "$meprefix"compress-texture "$pd_t" "$pd_dd" "$pd_i" "$pd_o".tmp.dds "$@"
 155                         pd_psnr_tmp=`compare -channel alpha -metric PSNR "$pd_i" "$pd_o".tmp.dds NULL:`
 156                         echo >&2 "$pd_dd: $pd_psnr_tmp dB"
 157                         pd_psnr_diff=`echo "($pd_psnr_tmp) - ($pd_psnr)" | bc -l`
 158                         case "$pd_psnr_diff" in
 159                                 -*|0)
 160                                         # tmp is smaller or equal
 161                                         # smaller PSNR is worse
 162                                         # no action
 163                                         ;;
 164                                 *)
 165                                         # tmp is larger
 166                                         # larger PSNR is better
 167                                         pd_psnr=$pd_psnr_tmp
 168                                         mv "$pd_o".tmp.dds "$pd_o"
 169                                         echo >&2 "PICKED (better)"
 170                                         ;;
 171                         esac
 172                 else
 173                         "$meprefix"compress-texture "$pd_t" "$pd_dd" "$pd_i" "$pd_o" "$@"
 174                         pd_psnr=`compare -channel alpha -metric PSNR "$pd_i" "$pd_o" NULL:`
 175                         echo >&2 "$pd_dd: $pd_psnr dB"
 176                         echo >&2 "PICKED (first)"
 177                 fi
 178         done
 179 }
 180
 181 reduce_jpeg2_dds()
 182 {
 183         i=$1; shift
 184         ia=$1; shift
 185         o=$1; shift; shift
 186         convert "$i" "$ia" -compose CopyOpacity -composite "$tmpdir/x.tga" && \
 187         pickdxta "$dds_tool" "$dds_sepalpha" "$tmpdir/x.tga" "$o" $1
 188 }
 189
 190 reduce_jpeg2_dds_premul()
 191 {
 192         i=$1; shift
 193         ia=$1; shift
 194         o=$1; shift; shift
 195         convert "$i" "$ia" -compose CopyOpacity -composite "$tmpdir/x.tga" && \
 196         pickdxta "$dds_tool" "$dds_prealpha" "$tmpdir/x.tga" "$o" $1
 197 }
 198
 199 reduce_jpeg2_jpeg2()
 200 {
 201         i=$1; shift
 202         ia=$1; shift
 203         o=$1; shift
 204         oa=$1; shift
 205         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 206                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
 207                         cp "$i" "$o"
 208                 fi
 209         else
 210                 return 1
 211         fi
 212         if convert "$ia" TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
 213                 if [ "`stat -c %s "$ia"`" -lt "`stat -c %s "$oa"`" ]; then
 214                         cp "$ia" "$oa"
 215                 fi
 216         else
 217                 return 1
 218         fi
 219 }
 220
 221 reduce_jpeg_jpeg()
 222 {
 223         i=$1; shift; shift
 224         o=$1; shift; shift
 225         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 226                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
 227                         cp "$i" "$o"
 228                 fi
 229         else
 230                 return 1
 231         fi
 232 }
 233
 234 reduce_ogg_ogg()
 235 {
 236         i=$1; shift; shift
 237         o=$1; shift; shift
 238         tags=`vorbiscomment -R -l "$i" || true`
 239         oggdec -o "$tmpdir/x.wav" "$i" && \
 240         oggenc -q"$1" -o "$o" "$tmpdir/x.wav"
 241         echo "$tags" | vorbiscomment -R -w "$o" || true
 242 }
 243
 244 reduce_wav_ogg()
 245 {
 246         i=$1; shift; shift
 247         o=$1; shift; shift
 248         oggenc -q"$1" -o "$o" "$i"
 249 }
 250
 251 reduce_rgba_dds()
 252 {
 253         i=$1; shift; shift
 254         o=$1; shift; shift
 255         convert "$i" "$tmpdir/x.tga" && \
 256         pickdxta "$dds_tool" "$dds_sepalpha" "$tmpdir/x.tga" "$o" $1
 257 }
 258
 259 reduce_rgba_dds_premul()
 260 {
 261         i=$1; shift; shift
 262         o=$1; shift; shift
 263         convert "$i" "$tmpdir/x.tga" && \
 264         pickdxta "$dds_tool" "$dds_prealpha" "$tmpdir/x.tga" "$o" $1
 265 }
 266
 267 reduce_rgba_jpeg2()
 268 {
 269         i=$1; shift; shift
 270         o=$1; shift
 271         oa=$1; shift
 272         if convert "$i" -alpha off TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 273                 :
 274         else
 275                 return 1
 276         fi
 277         if convert "$i" -alpha extract TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
 278                 :
 279         else
 280                 return 1
 281         fi
 282 }
 283
 284 reduce_rgb_dds()
 285 {
 286         i=$1; shift; shift
 287         o=$1; shift; shift
 288         convert "$i" "$tmpdir/x.tga" && \
 289         "$meprefix"compress-texture "$dds_tool" "$dds_noalpha" "$tmpdir/x.tga" "$o" $1
 290 }
 291
 292 reduce_rgb_jpeg()
 293 {
 294         i=$1; shift; shift
 295         o=$1; shift; shift
 296         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 297                 :
 298         else
 299                 return 1
 300         fi
 301 }
 302
 303 has_alpha()
 304 {
 305         i=$1; shift; shift
 306         o=$1; shift; shift
 307         if convert "$i" -depth 16 RGBA:- | perl -e 'while(read STDIN, $_, 8) { substr($_, 6, 2) eq "\xFF\xFF" or exit 1; } exit 0;'; then
 308                 # no alpha
 309                 : > "$o"
 310         else
 311                 # has alpha
 312                 echo yes > "$o"
 313         fi
 314 }
 315
 316 to_delete=
 317 for F in "$@"; do
 318         selfprofile prepareconvert
 319         f=${F%.*}
 320
 321         echo >&2 "Handling $F..."
 322         conv=false
 323         keep=false
 324         jqual_rgb=$jpeg_qual_rgb
 325         jqual_a=$jpeg_qual_a
 326
 327         will_jpeg=$do_jpeg
 328         will_dds=$do_dds
 329         will_ogg=$do_ogg
 330         case "$f" in
 331                 ./sounds/misc/talk*.wav) will_ogg=false ;; # engine "feature"
 332                 *_bump) will_dds=false ;;
 333                 ./models/player/*) will_dds=false ;;
 334                 ./models/sprites/*) will_dds=false ;;
 335                 ./textures/*) ;;
 336                 ./models/*) ;;
 337                 ./particles/*) ;;
 338                 ./progs/*) ;;
 339                 *)
 340                         # we can't DDS compress the 2D textures, sorry
 341                         # but JPEG is still fine
 342                         will_dds=false
 343                         ;;
 344         esac
 345
 346         # configure S2TC
 347         case "$f" in
 348                 *_norm)
 349                         export S2TC_COLORDIST_MODE=NORMALMAP
 350                         export S2TC_RANDOM_COLORS=256
 351                         export S2TC_REFINE_COLORS=LOOP
 352                         ;;
 353                 *)
 354                         export S2TC_COLORDIST_MODE=SRGB_MIXED
 355                         export S2TC_RANDOM_COLORS=64
 356                         export S2TC_REFINE_COLORS=LOOP
 357                         ;;
 358         esac
 359
 360         # for deluxemaps, lightmaps and normalmaps, enforce high jpeg quality (like on alpha channels)
 361         if [ "$jqual_a" -gt "$jqual_rgb" ]; then
 362                 case "$f" in
 363                         ./maps/*/lm_[0-9][0-9][0-9][13579]) # deluxemap
 364                                 jqual_rgb=$jqual_a
 365                                 ;;
 366                         ./maps/*/lm_[0-9][0-9][0-9][02468]) # lightmap
 367                                 jqual_rgb=$jqual_a
 368                                 ;;
 369                         *_norm) # normalmap
 370                                 jqual_rgb=$jqual_a
 371                                 ;;
 372                 esac
 373         fi
 374
 375         pm=
 376         case "$f" in
 377                 ./particles/particlefont) # particlefont uses premultiplied alpha
 378                         pm=_premul
 379                         ;;
 380         esac
 381
 382         if $do_jpeg_if_not_dds; then
 383                 if $will_dds; then
 384                         will_jpeg=false
 385                 else
 386                         will_jpeg=true
 387                 fi
 388         fi
 389         selfprofile startconvert
 390         case "$F" in
 391                 *_alpha.jpg)
 392                         # handle in *.jpg case
 393
 394                         # they always got converted, I assume
 395                         if $will_dds || $will_jpeg; then
 396                                 conv=true
 397                         fi
 398                         keep=$will_jpeg
 399                         ;;
 400                 *.jpg)
 401                         if [ -f "${f}_alpha.jpg" ]; then
 402                                 cached "$will_dds"  reduce_jpeg2_dds$pm "$F" "${f}_alpha.jpg" "dds/${f}.dds" ""               "$dds_flags"
 403                                 cached "$will_jpeg" reduce_jpeg2_jpeg2  "$F" "${f}_alpha.jpg" "$F"           "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
 404                         else
 405                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 406                                 cached "$will_jpeg" reduce_jpeg_jpeg    "$F" ""               "$F"           ""               "$jqual_rgb"
 407                         fi
 408                         ;;
 409                 *.png|*.tga)
 410                         cached true has_alpha "$F" "" "$F.hasalpha" ""
 411                         conv=false
 412                         if [ -s "$F.hasalpha" ]; then
 413                                 cached "$will_dds"  reduce_rgba_dds$pm  "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 414                                 cached "$will_jpeg" reduce_rgba_jpeg2   "$F" ""               "${f}.jpg"     "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
 415                         else
 416                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 417                                 cached "$will_jpeg" reduce_rgb_jpeg     "$F" ""               "${f}.jpg"     ""               "$jqual_rgb"
 418                         fi
 419                         rm -f "$F.hasalpha"
 420                         ;;
 421                 *.ogg)
 422                         cached "$will_ogg" reduce_ogg_ogg "$F" "" "$F" "" "$ogg_qual"
 423                         ;;
 424                 ./sound/misc/null.wav)
 425                         # never convert this one
 426                         ;;
 427                 *.wav)
 428                         cached "$will_ogg" reduce_wav_ogg "$F" "" "${f}.ogg" "" "$ogg_qual"
 429                         ;;
 430         esac
 431         selfprofile marktodelete
 432         if $del_src; then
 433                 if $conv; then
 434                         if ! $keep; then
 435                                 # FIXME can't have spaces in filenames that way
 436                                 to_delete="$to_delete $F"
 437                         fi
 438                 fi
 439         fi
 440         selfprofile symlinkfixing
 441         # fix up DDS paths by a symbolic link
 442         if [ -f "dds/${f}.dds" ]; then
 443                 if [ -z "${f##./textures/*}" ]; then
 444                         if [ -n "${f##./textures/*/*}" ]; then
 445                                 ln -snf "textures/${f#./textures/}.dds" "dds/${f#./textures/}.dds"
 446                         fi
 447                 fi
 448         fi
 449         selfprofile looping
 450 done
 451
 452 for F in $to_delete; do
 453         rm -f "$F"
 454 done
 455 selfprofile finished_time
 456 set | grep ^selfprofile_counter_ >&2