misc/tools/cached-converter.sh

   1 #!/bin/sh
   2
   3 set -e
   4
   5 : ${CACHEDIR:=$HOME/.xonotic-cached-converter}
   6 : ${do_jpeg:=true}
   7 : ${do_jpeg_if_not_dds:=false}
   8 : ${jpeg_qual_rgb:=95}
   9 : ${jpeg_qual_a:=99}
  10 : ${do_dds:=true}
  11 : ${dds_tool:=compressonator-dxtc}
  12 : ${do_ogg:=false}
  13 : ${ogg_qual:=1}
  14 : ${del_src:=false}
  15 : ${git_src_repo:=}
  16 : ${dds_noalpha:=dxt1}
  17 : ${dds_prealpha:=dxt2 dxt4}
  18 : ${dds_sepalpha:=dxt3 dxt5}
  19
  20 selfprofile_t0=`date +%s`
  21 selfprofile_step=init
  22 selfprofile()
  23 {
  24         selfprofile_t=`date +%s`
  25         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step+$selfprofile_t))"
  26         selfprofile_step=$1
  27         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step-$selfprofile_t))"
  28         selfprofile_t0=$selfprofile_t
  29 }
  30
  31 me=$0
  32 case "$me" in
  33         */*)
  34                 meprefix=${me%/*}/
  35                 ;;
  36         *)
  37                 meprefix=
  38                 ;;
  39 esac
  40
  41 tmpdir=`mktemp -d -t cached-converter.XXXXXX`
  42 trap 'exit 1' INT
  43 trap 'rm -rf "$tmpdir"' EXIT
  44
  45
  46 use_magnet_to_acquire_checksum_faster()
  47 #             ___________________
  48 #        ,--'' ~~~~~~~^^^~._     '.
  49 #    ,.-' ~~~~~~~~~~^^^^^~~~._._   \
  50 #    |   /^^^^^|    /^^^^^^^^\\ \   \
  51 #  ,/___  <  o>      <  (OO) > _     \
  52 # /'/,         |-         .       ----.\
  53 # |(|-'^^;,-  ,|     __    ^~~^^^^^^^; |\
  54 # \\`  |    <;_    __ |`---  ..-^^/- | ||
  55 #  \`-|Oq-.____`________~='^^|__,/  ' //
  56 #   \ || | |   |  |    \ ..-;|  /    '/
  57 #   | ||#|#|the|==|game!|'^` |/'    /'
  58 #   | \\\\^\***|***|    \ ,,;'     /
  59 #   |  `-=\_\__\___\__..-' ,.- - ,/
  60 #   | . `-_  ------   _,-'^-'^,-'
  61 #   | `-._________..--''^,-''^
  62 #   \             ,...-'^
  63 #    `----------'^              PROBLEM?
  64 {
  65         magnet=`GIT_DIR="$git_src_repo/.git" git ls-files -s "$1"`
  66         if [ -n "$magnet" ]; then
  67                 magnet=${magnet#* }
  68                 magnet=${magnet%% *}
  69                 sum=$sum$magnet
  70         else
  71                 sum=$sum`git hash-object "$1"`
  72         fi
  73 }
  74
  75 lastinfiles=
  76 lastinfileshash=
  77 cached()
  78 {
  79         flag=$1; shift
  80         method=$1; shift
  81         infile1=$1; shift
  82         infile2=$1; shift
  83         outfile1=$1; shift
  84         outfile2=$1; shift
  85         if ! $flag; then
  86                 return 0
  87         fi
  88         #sleep 0.25
  89         if [ x"$infile1" = x"$outfile1" ]; then
  90                 keep=true
  91         fi
  92         options=`echo "$*" | git hash-object --stdin`
  93         selfprofile convert_findchecksum
  94         if [ x"$infile1/../$infile2" = x"$lastinfiles" ]; then
  95                 sum=$lastinfileshash
  96         else
  97                 evil=false
  98                 for infile in "$infile1" "$infile2"; do
  99                         case "$infile" in
 100                                 */background_l2.tga|*/background_ingame_l2.tga)
 101                                         evil=true
 102                                         ;;
 103                         esac
 104                 done
 105                 if [ -n "$git_src_repo" ] && ! $evil; then
 106                         sum=
 107                         use_magnet_to_acquire_checksum_faster "${infile1#./}"
 108                         if [ -n "$infile2" ]; then
 109                                 use_magnet_to_acquire_checksum_faster "${infile2#./}"
 110                         fi
 111                 else
 112                         sum=`git hash-object "$infile1"`
 113                         if [ -n "$infile2" ]; then
 114                                 sum=$sum`git hash-object "$infile2"`
 115                         fi
 116                 fi
 117                 lastinfileshash=$sum
 118         fi
 119         selfprofile convert_makecachedir
 120         mkdir -p "$CACHEDIR/$method-$options"
 121         name1="$CACHEDIR/$method-$options/$sum-1.${outfile1##*.}"
 122         [ -z "$outfile2" ] || name2="$CACHEDIR/$method-$options/$sum-2.${outfile2##*.}"
 123         tempfile1="${name1%/*}/new-${name1##*/}"
 124         [ -z "$outfile2" ] || tempfile2="${name2%/*}/new-${name2##*/}"
 125         if [ -f "$name1" ] && { [ -z "$outfile2" ] || [ -f "$name2" ]; }; then
 126                 selfprofile convert_copyoutput
 127                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
 128                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
 129                 conv=true
 130         elif selfprofile convert_makeoutput; "$method" "$infile1" "$infile2" "$tempfile1" "$tempfile2" "$@"; then
 131                 mv "$tempfile1" "$name1"
 132                 [ -z "$outfile2" ] || mv "$tempfile2" "$name2"
 133                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
 134                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
 135                 conv=true
 136         else
 137                 selfprofile convert_cleartemp
 138                 rm -f "$tempfile1"
 139                 rm -f "$tempfile2"
 140                 selfprofile convert_finished
 141                 exit 1
 142         fi
 143         selfprofile convert_finished
 144 }
 145
 146 pickdxta()
 147 {
 148         pd_t=$1; shift
 149         pd_d=$1; shift
 150         pd_i=$1; shift
 151         pd_o=$1; shift
 152         for pd_dd in $pd_d; do
 153                 if [ -f "$pd_o" ]; then
 154                         "$meprefix"compress-texture "$pd_t" "$pd_dd" "$pd_i" "$pd_o".tmp.dds "$@"
 155                         pd_psnr_tmp=`compare -channel alpha -metric PSNR "$pd_i" "$pd_o".tmp.dds NULL: 2>&1`
 156                         case "$pd_psnr_tmp" in
 157                                 [0-9.]*)
 158                                         ;;
 159                                 *)
 160                                         pd_psnr_tmp=999.9
 161                                         ;;
 162                         esac
 163                         echo >&2 "$pd_dd: $pd_psnr_tmp dB"
 164                         pd_psnr_diff=`echo "($pd_psnr_tmp) - ($pd_psnr)" | bc -l`
 165                         case "$pd_psnr_diff" in
 166                                 -*|0)
 167                                         # tmp is smaller or equal
 168                                         # smaller PSNR is worse
 169                                         # no action
 170                                         ;;
 171                                 *)
 172                                         # tmp is larger
 173                                         # larger PSNR is better
 174                                         pd_psnr=$pd_psnr_tmp
 175                                         mv "$pd_o".tmp.dds "$pd_o"
 176                                         echo >&2 "PICKED (better)"
 177                                         ;;
 178                         esac
 179                 else
 180                         "$meprefix"compress-texture "$pd_t" "$pd_dd" "$pd_i" "$pd_o" "$@"
 181                         pd_psnr=`compare -channel alpha -metric PSNR "$pd_i" "$pd_o" NULL: 2>&1`
 182                         case "$pd_psnr" in
 183                                 [0-9.]*)
 184                                         ;;
 185                                 *)
 186                                         pd_psnr=999.9
 187                                         ;;
 188                         esac
 189                         echo >&2 "$pd_dd: $pd_psnr dB"
 190                         echo >&2 "PICKED (first)"
 191                 fi
 192         done
 193 }
 194
 195 reduce_jpeg2_dds()
 196 {
 197         i=$1; shift
 198         ia=$1; shift
 199         o=$1; shift; shift
 200         convert "$i" "$ia" -compose CopyOpacity -composite "$tmpdir/x.tga" && \
 201         pickdxta "$dds_tool" "$dds_sepalpha" "$tmpdir/x.tga" "$o" $1
 202 }
 203
 204 reduce_jpeg2_dds_premul()
 205 {
 206         i=$1; shift
 207         ia=$1; shift
 208         o=$1; shift; shift
 209         convert "$i" "$ia" -compose CopyOpacity -composite "$tmpdir/x.tga" && \
 210         pickdxta "$dds_tool" "$dds_prealpha" "$tmpdir/x.tga" "$o" $1
 211 }
 212
 213 reduce_jpeg2_jpeg2()
 214 {
 215         i=$1; shift
 216         ia=$1; shift
 217         o=$1; shift
 218         oa=$1; shift
 219         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 220                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
 221                         cp "$i" "$o"
 222                 fi
 223         else
 224                 return 1
 225         fi
 226         if convert "$ia" TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
 227                 if [ "`stat -c %s "$ia"`" -lt "`stat -c %s "$oa"`" ]; then
 228                         cp "$ia" "$oa"
 229                 fi
 230         else
 231                 return 1
 232         fi
 233 }
 234
 235 reduce_jpeg_jpeg()
 236 {
 237         i=$1; shift; shift
 238         o=$1; shift; shift
 239         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 240                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
 241                         cp "$i" "$o"
 242                 fi
 243         else
 244                 return 1
 245         fi
 246 }
 247
 248 reduce_ogg_ogg()
 249 {
 250         i=$1; shift; shift
 251         o=$1; shift; shift
 252         tags=`vorbiscomment -R -l "$i" || true`
 253         oggdec -o "$tmpdir/x.wav" "$i" && \
 254         oggenc -q"$1" -o "$o" "$tmpdir/x.wav"
 255         echo "$tags" | vorbiscomment -R -w "$o" || true
 256 }
 257
 258 reduce_wav_ogg()
 259 {
 260         i=$1; shift; shift
 261         o=$1; shift; shift
 262         oggenc -q"$1" -o "$o" "$i"
 263 }
 264
 265 reduce_rgba_dds()
 266 {
 267         i=$1; shift; shift
 268         o=$1; shift; shift
 269         convert "$i" "$tmpdir/x.tga" && \
 270         pickdxta "$dds_tool" "$dds_sepalpha" "$tmpdir/x.tga" "$o" $1
 271 }
 272
 273 reduce_rgba_dds_premul()
 274 {
 275         i=$1; shift; shift
 276         o=$1; shift; shift
 277         convert "$i" "$tmpdir/x.tga" && \
 278         pickdxta "$dds_tool" "$dds_prealpha" "$tmpdir/x.tga" "$o" $1
 279 }
 280
 281 reduce_rgba_jpeg2()
 282 {
 283         i=$1; shift; shift
 284         o=$1; shift
 285         oa=$1; shift
 286         if convert "$i" -alpha off TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 287                 :
 288         else
 289                 return 1
 290         fi
 291         if convert "$i" -alpha extract TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
 292                 :
 293         else
 294                 return 1
 295         fi
 296 }
 297
 298 reduce_rgb_dds()
 299 {
 300         i=$1; shift; shift
 301         o=$1; shift; shift
 302         convert "$i" "$tmpdir/x.tga" && \
 303         "$meprefix"compress-texture "$dds_tool" "$dds_noalpha" "$tmpdir/x.tga" "$o" $1
 304 }
 305
 306 reduce_rgb_jpeg()
 307 {
 308         i=$1; shift; shift
 309         o=$1; shift; shift
 310         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 311                 :
 312         else
 313                 return 1
 314         fi
 315 }
 316
 317 has_alpha()
 318 {
 319         i=$1; shift; shift
 320         o=$1; shift; shift
 321         if convert "$i" -depth 16 RGBA:- | perl -e 'while(read STDIN, $_, 8) { substr($_, 6, 2) eq "\xFF\xFF" or exit 1; } exit 0;'; then
 322                 # no alpha
 323                 : > "$o"
 324         else
 325                 # has alpha
 326                 echo yes > "$o"
 327         fi
 328 }
 329
 330 to_delete=
 331 for F in "$@"; do
 332         selfprofile prepareconvert
 333         f=${F%.*}
 334
 335         echo >&2 "Handling $F..."
 336         conv=false
 337         keep=false
 338         jqual_rgb=$jpeg_qual_rgb
 339         jqual_a=$jpeg_qual_a
 340
 341         will_jpeg=$do_jpeg
 342         will_dds=$do_dds
 343         will_ogg=$do_ogg
 344         case "$f" in
 345                 ./sounds/misc/talk*.wav) will_ogg=false ;; # engine "feature"
 346                 *_bump) will_dds=false ;;
 347                 ./models/player/*) will_dds=false ;;
 348                 ./models/sprites/*) will_dds=false ;;
 349                 ./textures/*) ;;
 350                 ./models/*) ;;
 351                 ./particles/*) ;;
 352                 ./progs/*) ;;
 353                 *)
 354                         # we can't DDS compress the 2D textures, sorry
 355                         # but JPEG is still fine
 356                         will_dds=false
 357                         ;;
 358         esac
 359
 360         # configure S2TC
 361         case "$f" in
 362                 *_norm)
 363                         export S2TC_COLORDIST_MODE=NORMALMAP
 364                         export S2TC_RANDOM_COLORS=256
 365                         export S2TC_REFINE_COLORS=LOOP
 366                         ;;
 367                 *)
 368                         export S2TC_COLORDIST_MODE=SRGB_MIXED
 369                         export S2TC_RANDOM_COLORS=64
 370                         export S2TC_REFINE_COLORS=LOOP
 371                         ;;
 372         esac
 373
 374         # for deluxemaps, lightmaps and normalmaps, enforce high jpeg quality (like on alpha channels)
 375         if [ "$jqual_a" -gt "$jqual_rgb" ]; then
 376                 case "$f" in
 377                         ./maps/*/lm_[0-9][0-9][0-9][13579]) # deluxemap
 378                                 jqual_rgb=$jqual_a
 379                                 ;;
 380                         ./maps/*/lm_[0-9][0-9][0-9][02468]) # lightmap
 381                                 jqual_rgb=$jqual_a
 382                                 ;;
 383                         *_norm) # normalmap
 384                                 jqual_rgb=$jqual_a
 385                                 ;;
 386                 esac
 387         fi
 388
 389         pm=
 390         case "$f" in
 391                 ./particles/particlefont) # particlefont uses premultiplied alpha
 392                         pm=_premul
 393                         ;;
 394         esac
 395
 396         if $do_jpeg_if_not_dds; then
 397                 if $will_dds; then
 398                         will_jpeg=false
 399                 else
 400                         will_jpeg=true
 401                 fi
 402         fi
 403         selfprofile startconvert
 404         case "$F" in
 405                 *_alpha.jpg)
 406                         # handle in *.jpg case
 407
 408                         # they always got converted, I assume
 409                         if $will_dds || $will_jpeg; then
 410                                 conv=true
 411                         fi
 412                         keep=$will_jpeg
 413                         ;;
 414                 *.jpg)
 415                         if [ -f "${f}_alpha.jpg" ]; then
 416                                 cached "$will_dds"  reduce_jpeg2_dds$pm "$F" "${f}_alpha.jpg" "dds/${f}.dds" ""               "$dds_flags"
 417                                 cached "$will_jpeg" reduce_jpeg2_jpeg2  "$F" "${f}_alpha.jpg" "$F"           "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
 418                         else
 419                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 420                                 cached "$will_jpeg" reduce_jpeg_jpeg    "$F" ""               "$F"           ""               "$jqual_rgb"
 421                         fi
 422                         ;;
 423                 *.png|*.tga)
 424                         cached true has_alpha "$F" "" "$F.hasalpha" ""
 425                         conv=false
 426                         if [ -s "$F.hasalpha" ]; then
 427                                 cached "$will_dds"  reduce_rgba_dds$pm  "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 428                                 cached "$will_jpeg" reduce_rgba_jpeg2   "$F" ""               "${f}.jpg"     "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
 429                         else
 430                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 431                                 cached "$will_jpeg" reduce_rgb_jpeg     "$F" ""               "${f}.jpg"     ""               "$jqual_rgb"
 432                         fi
 433                         rm -f "$F.hasalpha"
 434                         ;;
 435                 *.ogg)
 436                         cached "$will_ogg" reduce_ogg_ogg "$F" "" "$F" "" "$ogg_qual"
 437                         ;;
 438                 ./sound/misc/null.wav)
 439                         # never convert this one
 440                         ;;
 441                 *.wav)
 442                         cached "$will_ogg" reduce_wav_ogg "$F" "" "${f}.ogg" "" "$ogg_qual"
 443                         ;;
 444         esac
 445         selfprofile marktodelete
 446         if $del_src; then
 447                 if $conv; then
 448                         if ! $keep; then
 449                                 # FIXME can't have spaces in filenames that way
 450                                 to_delete="$to_delete $F"
 451                         fi
 452                 fi
 453         fi
 454         selfprofile symlinkfixing
 455         # fix up DDS paths by a symbolic link
 456         if [ -f "dds/${f}.dds" ]; then
 457                 if [ -z "${f##./textures/*}" ]; then
 458                         if [ -n "${f##./textures/*/*}" ]; then
 459                                 ln -snf "textures/${f#./textures/}.dds" "dds/${f#./textures/}.dds"
 460                         fi
 461                 fi
 462         fi
 463         selfprofile looping
 464 done
 465
 466 for F in $to_delete; do
 467         rm -f "$F"
 468 done
 469 selfprofile finished_time
 470 set | grep ^selfprofile_counter_ >&2