misc/tools/cached-converter.sh

   1 #!/bin/sh
   2
   3 set -e
   4
   5 : ${CACHEDIR:=$HOME/.xonotic-cached-converter}
   6 : ${do_jpeg:=true}
   7 : ${do_jpeg_if_not_dds:=false}
   8 : ${jpeg_qual_rgb:=95}
   9 : ${jpeg_qual_a:=99}
  10 : ${do_webp:=false}
  11 : ${do_webp_if_not_dds:=false}
  12 : ${webp_flags_lq:=-lossless -q 100}
  13 : ${webp_flags_hq:=-lossless -q 100}
  14 : ${webp_flags_alq:=-lossless -q 100 -q_alpha 100}
  15 : ${webp_flags_ahq:=-lossless -q 100 -q_alpha 100}
  16 : ${do_dds:=true}
  17 : ${dds_tool:=s2tc}
  18 : ${do_ogg:=false}
  19 : ${ogg_ogg:=true}
  20 : ${ogg_qual:=2}
  21 : ${del_src:=false}
  22 : ${git_src_repo:=}
  23 : ${dds_noalpha:=dxt1}
  24 : ${dds_prealpha:=dxt2 dxt4}
  25 : ${dds_sepalpha:=dxt3 dxt5}
  26
  27 selfprofile_t0=`date +%s`
  28 selfprofile_step=init
  29 selfprofile()
  30 {
  31         selfprofile_t=`date +%s`
  32         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step+$selfprofile_t))"
  33         selfprofile_step=$1
  34         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step-$selfprofile_t))"
  35         selfprofile_t0=$selfprofile_t
  36 }
  37
  38 me=$0
  39 case "$me" in
  40         */*)
  41                 meprefix=${me%/*}/
  42                 ;;
  43         *)
  44                 meprefix=
  45                 ;;
  46 esac
  47
  48 tmpdir=`mktemp -d -t cached-converter.XXXXXX`
  49 trap 'exit 1' INT
  50 trap 'rm -rf "$tmpdir"' EXIT
  51
  52
  53 use_magnet_to_acquire_checksum_faster()
  54 #             ___________________
  55 #        ,--'' ~~~~~~~^^^~._     '.
  56 #    ,.-' ~~~~~~~~~~^^^^^~~~._._   \
  57 #    |   /^^^^^|    /^^^^^^^^\\ \   \
  58 #  ,/___  <  o>      <  (OO) > _     \
  59 # /'/,         |-         .       ----.\
  60 # |(|-'^^;,-  ,|     __    ^~~^^^^^^^; |\
  61 # \\`  |    <;_    __ |`---  ..-^^/- | ||
  62 #  \`-|Oq-.____`________~='^^|__,/  ' //
  63 #   \ || | |   |  |    \ ..-;|  /    '/
  64 #   | ||#|#|the|==|game!|'^` |/'    /'
  65 #   | \\\\^\***|***|    \ ,,;'     /
  66 #   |  `-=\_\__\___\__..-' ,.- - ,/
  67 #   | . `-_  ------   _,-'^-'^,-'
  68 #   | `-._________..--''^,-''^
  69 #   \             ,...-'^
  70 #    `----------'^              PROBLEM?
  71 {
  72         magnet=`GIT_DIR="$git_src_repo/.git" git ls-files -s "$1"`
  73         if [ -n "$magnet" ]; then
  74                 magnet=${magnet#* }
  75                 magnet=${magnet%% *}
  76                 echo "$magnet"
  77         else
  78                 git hash-object "$1"
  79         fi
  80 }
  81
  82 lastinfiles=
  83 lastinfileshash=
  84 acquire_checksum()
  85 {
  86         if [ x"$1/../$2" = x"$lastinfiles" ]; then
  87                 _a_s=$lastinfileshash
  88         else
  89                 _a_e=false
  90                 for _a_f in "$1" "$2"; do
  91                         case "$_a_f" in
  92                                 */background_l2.tga|*/background_ingame_l2.tga)
  93                                         _a_e=true
  94                                         ;;
  95                         esac
  96                 done
  97                 if [ -n "$git_src_repo" ] && ! $_a_e; then
  98                         _a_s=`use_magnet_to_acquire_checksum_faster "${1#./}"`
  99                         if [ -n "$2" ]; then
 100                                 _a_s=$_a_s`use_magnet_to_acquire_checksum_faster "${2#./}"`
 101                         fi
 102                 else
 103                         _a_s=`git hash-object "$1"`
 104                         if [ -n "$2" ]; then
 105                                 _a_s=$_a_s`git hash-object "$2"`
 106                         fi
 107                 fi
 108                 lastinfileshash=$_a_s
 109                 lastinfiles="$1/../$2"
 110         fi
 111         echo "$_a_s"
 112 }
 113
 114 cached()
 115 {
 116         flag=$1; shift
 117         method=$1; shift
 118         infile1=$1; shift
 119         infile2=$1; shift
 120         outfile1=$1; shift
 121         outfile2=$1; shift
 122         if ! $flag; then
 123                 return 0
 124         fi
 125         #sleep 0.25
 126         if [ x"$infile1" = x"$outfile1" ]; then
 127                 keep=true
 128         fi
 129         options=`echo "$*" | git hash-object --stdin`
 130         selfprofile convert_findchecksum
 131         sum=`acquire_checksum "$infile1" "$infile2"`
 132         selfprofile convert_makecachedir
 133         mkdir -p "$CACHEDIR/$method-$options"
 134         name1="$CACHEDIR/$method-$options/$sum-1.${outfile1##*.}"
 135         [ -z "$outfile2" ] || name2="$CACHEDIR/$method-$options/$sum-2.${outfile2##*.}"
 136         tempfile1="${name1%/*}/new-${name1##*/}"
 137         [ -z "$outfile2" ] || tempfile2="${name2%/*}/new-${name2##*/}"
 138         if [ -f "$name1" ] && { [ -z "$outfile2" ] || [ -f "$name2" ]; }; then
 139                 selfprofile convert_copyoutput
 140                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
 141                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
 142                 conv=true
 143         elif selfprofile convert_makeoutput; "$method" "$infile1" "$infile2" "$tempfile1" "$tempfile2" "$@"; then
 144                 mv "$tempfile1" "$name1"
 145                 [ -z "$outfile2" ] || mv "$tempfile2" "$name2"
 146                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
 147                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
 148                 conv=true
 149         else
 150                 selfprofile convert_cleartemp
 151                 rm -f "$tempfile1"
 152                 rm -f "$tempfile2"
 153                 selfprofile convert_finished
 154                 exit 1
 155         fi
 156         selfprofile convert_finished
 157 }
 158
 159 pickdxta()
 160 {
 161         pd_t=$1; shift
 162         pd_d=$1; shift
 163         pd_i=$1; shift
 164         pd_o=$1; shift
 165         for pd_dd in $pd_d; do
 166                 if [ -f "$pd_o" ]; then
 167                         "$meprefix"compress-texture "$pd_t" "$pd_dd" "$pd_i" "$pd_o".tmp.dds "$@"
 168                         pd_psnr_tmp=`compare -channel alpha -metric PSNR "$pd_i" "$pd_o".tmp.dds NULL: 2>&1`
 169                         case "$pd_psnr_tmp" in
 170                                 [0-9.]*)
 171                                         ;;
 172                                 *)
 173                                         pd_psnr_tmp=999.9
 174                                         ;;
 175                         esac
 176                         echo >&2 "$pd_dd: $pd_psnr_tmp dB"
 177                         pd_psnr_diff=`echo "($pd_psnr_tmp) - ($pd_psnr)" | bc -l`
 178                         case "$pd_psnr_diff" in
 179                                 -*|0)
 180                                         # tmp is smaller or equal
 181                                         # smaller PSNR is worse
 182                                         # no action
 183                                         ;;
 184                                 *)
 185                                         # tmp is larger
 186                                         # larger PSNR is better
 187                                         pd_psnr=$pd_psnr_tmp
 188                                         mv "$pd_o".tmp.dds "$pd_o"
 189                                         echo >&2 "PICKED (better)"
 190                                         ;;
 191                         esac
 192                 else
 193                         "$meprefix"compress-texture "$pd_t" "$pd_dd" "$pd_i" "$pd_o" "$@"
 194                         pd_psnr=`compare -channel alpha -metric PSNR "$pd_i" "$pd_o" NULL: 2>&1`
 195                         case "$pd_psnr" in
 196                                 [0-9.]*)
 197                                         ;;
 198                                 *)
 199                                         pd_psnr=999.9
 200                                         ;;
 201                         esac
 202                         echo >&2 "$pd_dd: $pd_psnr dB"
 203                         echo >&2 "PICKED (first)"
 204                 fi
 205         done
 206 }
 207
 208 reduce_jpeg2_dds()
 209 {
 210         i=$1; shift
 211         ia=$1; shift
 212         o=$1; shift; shift
 213         convert "$i" "$ia" -compose CopyOpacity -composite -type TrueColorMatte "$tmpdir/x.tga" && \
 214         pickdxta "$dds_tool" "$dds_sepalpha" "$tmpdir/x.tga" "$o" $1
 215 }
 216
 217 reduce_jpeg2_dds_premul()
 218 {
 219         i=$1; shift
 220         ia=$1; shift
 221         o=$1; shift; shift
 222         convert "$i" "$ia" -compose CopyOpacity -composite -type TrueColorMatte "$tmpdir/x.tga" && \
 223         pickdxta "$dds_tool" "$dds_prealpha" "$tmpdir/x.tga" "$o" $1
 224 }
 225
 226 reduce_jpeg2_jpeg2()
 227 {
 228         i=$1; shift
 229         ia=$1; shift
 230         o=$1; shift
 231         oa=$1; shift
 232         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 233                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
 234                         cp "$i" "$o"
 235                 fi
 236         else
 237                 return 1
 238         fi
 239         if convert "$ia" TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
 240                 if [ "`stat -c %s "$ia"`" -lt "`stat -c %s "$oa"`" ]; then
 241                         cp "$ia" "$oa"
 242                 fi
 243         else
 244                 return 1
 245         fi
 246 }
 247
 248 reduce_jpeg2_webp()
 249 {
 250         i=$1; shift
 251         ia=$1; shift
 252         o=$1; shift; shift
 253         # this one MUST run
 254         convert "$i" "$ia" -compose CopyOpacity -composite -type TrueColorMatte "$tmpdir/x.png" && \
 255         cwebp $1 "$tmpdir/x.png" -o "$o"
 256 }
 257
 258 reduce_jpeg_jpeg()
 259 {
 260         i=$1; shift; shift
 261         o=$1; shift; shift
 262         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 263                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
 264                         cp "$i" "$o"
 265                 fi
 266         else
 267                 return 1
 268         fi
 269 }
 270
 271 reduce_ogg_ogg()
 272 {
 273         i=$1; shift; shift
 274         o=$1; shift; shift
 275         tags=`vorbiscomment -R -l "$i" || true`
 276         oggdec -o "$tmpdir/x.wav" "$i" && \
 277         oggenc -q"$1" -o "$o" "$tmpdir/x.wav"
 278         echo "$tags" | vorbiscomment -R -w "$o" || true
 279 }
 280
 281 reduce_wav_ogg()
 282 {
 283         i=$1; shift; shift
 284         o=$1; shift; shift
 285         oggenc -q"$1" -o "$o" "$i"
 286 }
 287
 288 reduce_rgba_dds()
 289 {
 290         i=$1; shift; shift
 291         o=$1; shift; shift
 292         convert "$i" -type TrueColorMatte "$tmpdir/x.tga" && \
 293         pickdxta "$dds_tool" "$dds_sepalpha" "$tmpdir/x.tga" "$o" $1
 294 }
 295
 296 reduce_rgba_dds_premul()
 297 {
 298         i=$1; shift; shift
 299         o=$1; shift; shift
 300         convert "$i" -type TrueColorMatte "$tmpdir/x.tga" && \
 301         pickdxta "$dds_tool" "$dds_prealpha" "$tmpdir/x.tga" "$o" $1
 302 }
 303
 304 reduce_rgba_jpeg2()
 305 {
 306         i=$1; shift; shift
 307         o=$1; shift
 308         oa=$1; shift
 309         if convert "$i" -alpha off TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 310                 :
 311         else
 312                 return 1
 313         fi
 314         if convert "$i" -alpha extract TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
 315                 :
 316         else
 317                 return 1
 318         fi
 319 }
 320
 321 reduce_rgb_dds()
 322 {
 323         i=$1; shift; shift
 324         o=$1; shift; shift
 325         convert "$i" -type TrueColor "$tmpdir/x.tga" && \
 326         "$meprefix"compress-texture "$dds_tool" "$dds_noalpha" "$tmpdir/x.tga" "$o" $1
 327 }
 328
 329 reduce_rgb_jpeg()
 330 {
 331         i=$1; shift; shift
 332         o=$1; shift; shift
 333         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 334                 :
 335         else
 336                 return 1
 337         fi
 338 }
 339
 340 reduce_rgba_webp()
 341 {
 342         i=$1; shift; shift
 343         o=$1; shift; shift
 344         convert "$i" "$tmpdir/x.png" && \
 345         cwebp $1 "$tmpdir/x.png" -o "$o"
 346 }
 347
 348 has_alpha()
 349 {
 350         i=$1; shift; shift
 351         o=$1; shift; shift
 352         if convert "$i" -depth 16 RGBA:- | perl -e 'while(read STDIN, $_, 8) { substr($_, 6, 2) eq "\xFF\xFF" or exit 1; } exit 0;'; then
 353                 # no alpha
 354                 : > "$o"
 355         else
 356                 # has alpha
 357                 echo yes > "$o"
 358         fi
 359 }
 360
 361 to_delete=
 362 for F in "$@"; do
 363         selfprofile prepareconvert
 364         f=${F%.*}
 365
 366         echo >&2 "Handling $F..."
 367         conv=false
 368         keep=false
 369         jqual_rgb=$jpeg_qual_rgb
 370         jqual_a=$jpeg_qual_a
 371         webp_mode=lq
 372
 373         will_jpeg=$do_jpeg
 374         will_webp=$do_webp
 375         will_dds=$do_dds
 376         will_ogg=$do_ogg
 377         if ! $ogg_ogg; then
 378                 case "$F" in
 379                         *.ogg) will_ogg=false ;;
 380                 esac
 381         fi
 382         case "$F" in
 383                 ./sound/misc/talk*.wav) will_ogg=false ;; # engine "feature"
 384                 *_bump.*) will_dds=false ;;
 385                 ./models/player/*) will_dds=false ;;
 386                 ./models/sprites/*) will_dds=false ;;
 387                 ./models/*) ;;
 388                 ./textures/*) ;;
 389                 ./models/*) ;;
 390                 ./particles/*) ;;
 391                 ./progs/*) ;;
 392                 *)
 393                         # we can't DDS compress the 2D textures, sorry
 394                         # but JPEG is still fine
 395                         will_dds=false
 396                         ;;
 397         esac
 398
 399         # configure S2TC
 400         case "$f" in
 401                 *_norm)
 402                         export S2TC_COLORDIST_MODE=NORMALMAP
 403                         export S2TC_RANDOM_COLORS=256
 404                         export S2TC_REFINE_COLORS=LOOP
 405                         ;;
 406                 *)
 407                         export S2TC_COLORDIST_MODE=SRGB_MIXED
 408                         export S2TC_RANDOM_COLORS=64
 409                         export S2TC_REFINE_COLORS=LOOP
 410                         ;;
 411         esac
 412
 413         # for deluxemaps, lightmaps and normalmaps, enforce high jpeg quality (like on alpha channels)
 414         if [ "$jqual_a" -gt "$jqual_rgb" ]; then
 415                 case "$f" in
 416                         ./maps/*/lm_[0-9][0-9][0-9][13579]) # deluxemap
 417                                 jqual_rgb=$jqual_a
 418                                 webp_mode=hq
 419                                 ;;
 420                         ./maps/*/lm_[0-9][0-9][0-9][02468]) # lightmap
 421                                 jqual_rgb=$jqual_a
 422                                 webp_mode=hq
 423                                 ;;
 424                         *_norm) # normalmap
 425                                 jqual_rgb=$jqual_a
 426                                 webp_mode=hq
 427                                 ;;
 428                 esac
 429         fi
 430
 431         pm=
 432         case "$f" in
 433                 ./particles/particlefont) # particlefont uses premultiplied alpha
 434                         pm=_premul
 435                         ;;
 436         esac
 437
 438         if $do_jpeg_if_not_dds; then
 439                 if $will_dds; then
 440                         will_jpeg=false
 441                 else
 442                         will_jpeg=true
 443                 fi
 444         fi
 445         if $do_webp_if_not_dds; then
 446                 if $will_dds; then
 447                         will_webp=false
 448                 else
 449                         will_webp=true
 450                 fi
 451         fi
 452         selfprofile startconvert
 453         case "$F" in
 454                 *_alpha.jpg)
 455                         # handle in *.jpg case
 456
 457                         # they always got converted, I assume
 458                         if $will_dds || $will_jpeg || $will_webp; then
 459                                 conv=true
 460                         fi
 461                         keep=$will_jpeg
 462                         ;;
 463                 *.jpg)
 464                         if [ -f "${f}_alpha.jpg" ]; then
 465                                 cached "$will_dds"  reduce_jpeg2_dds$pm "$F" "${f}_alpha.jpg" "dds/${f}.dds" ""               "$dds_flags"
 466                                 cached "$will_jpeg" reduce_jpeg2_jpeg2  "$F" "${f}_alpha.jpg" "$F"           "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
 467                                 #eval wflags=\$webp_flags_${webp_mode}a
 468                                 #cached "$will_webp" reduce_jpeg2_webp   "$F" "${f}_alpha.jpg" "${f}.webp"    ""               "$wflags"
 469                         else
 470                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 471                                 cached "$will_jpeg" reduce_jpeg_jpeg    "$F" ""               "$F"           ""               "$jqual_rgb"
 472                                 #eval wflags=\$webp_flags_${webp_mode}
 473                                 #cached "$will_webp" reduce_rgba_webp    "$F" ""               "${f}.webp"    ""               "$wflags"
 474                         fi
 475                         ;;
 476                 *.png|*.tga|*.webp)
 477                         cached true has_alpha "$F" "" "$F.hasalpha" ""
 478                         conv=false
 479                         if [ -s "$F.hasalpha" ]; then
 480                                 cached "$will_dds"  reduce_rgba_dds$pm  "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 481                                 cached "$will_jpeg" reduce_rgba_jpeg2   "$F" ""               "${f}.jpg"     "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
 482                                 eval wflags=\$webp_flags_${webp_mode}a
 483                                 cached "$will_webp" reduce_rgba_webp    "$F" ""               "${f}.webp"    ""               "$wflags"
 484                         else
 485                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 486                                 cached "$will_jpeg" reduce_rgb_jpeg     "$F" ""               "${f}.jpg"     ""               "$jqual_rgb"
 487                                 eval wflags=\$webp_flags_${webp_mode}
 488                                 cached "$will_webp" reduce_rgba_webp    "$F" ""               "${f}.webp"    ""               "$wflags"
 489                         fi
 490                         rm -f "$F.hasalpha"
 491                         ;;
 492                 *.ogg)
 493                         cached "$will_ogg" reduce_ogg_ogg "$F" "" "$F" "" "$ogg_qual"
 494                         ;;
 495                 ./sound/misc/null.wav)
 496                         # never convert this one
 497                         ;;
 498                 *.wav)
 499                         cached "$will_ogg" reduce_wav_ogg "$F" "" "${f}.ogg" "" "$ogg_qual"
 500                         ;;
 501         esac
 502         selfprofile marktodelete
 503         if $del_src; then
 504                 if $conv; then
 505                         if ! $keep; then
 506                                 # FIXME can't have spaces in filenames that way
 507                                 to_delete="$to_delete $F"
 508                         fi
 509                 fi
 510         fi
 511         selfprofile symlinkfixing
 512         # fix up DDS paths by a symbolic link
 513         if [ -f "dds/${f}.dds" ]; then
 514                 if [ -z "${f##./textures/*}" ]; then
 515                         if [ -n "${f##./textures/*/*}" ]; then
 516                                 ln -snf "textures/${f#./textures/}.dds" "dds/${f#./textures/}.dds"
 517                         fi
 518                 fi
 519         fi
 520         selfprofile looping
 521 done
 522
 523 for F in $to_delete; do
 524         rm -f "$F"
 525 done
 526 selfprofile finished_time
 527 set | grep ^selfprofile_counter_ >&2