misc/tools/cached-converter.sh

   1 #!/bin/sh
   2
   3 set -e
   4
   5 : ${CACHEDIR:=$HOME/.xonotic-cached-converter}
   6 : ${do_jpeg:=true}
   7 : ${do_jpeg_if_not_dds:=false}
   8 : ${jpeg_qual_rgb:=95}
   9 : ${jpeg_qual_a:=99}
  10 : ${do_dds:=true}
  11 : ${dds_tool:=compressonator-dxtc}
  12 : ${do_ogg:=false}
  13 : ${ogg_ogg:=true}
  14 : ${ogg_qual:=2}
  15 : ${del_src:=false}
  16 : ${git_src_repo:=}
  17 : ${dds_noalpha:=dxt1}
  18 : ${dds_prealpha:=dxt2 dxt4}
  19 : ${dds_sepalpha:=dxt3 dxt5}
  20
  21 selfprofile_t0=`date +%s`
  22 selfprofile_step=init
  23 selfprofile()
  24 {
  25         selfprofile_t=`date +%s`
  26         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step+$selfprofile_t))"
  27         selfprofile_step=$1
  28         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step-$selfprofile_t))"
  29         selfprofile_t0=$selfprofile_t
  30 }
  31
  32 me=$0
  33 case "$me" in
  34         */*)
  35                 meprefix=${me%/*}/
  36                 ;;
  37         *)
  38                 meprefix=
  39                 ;;
  40 esac
  41
  42 tmpdir=`mktemp -d -t cached-converter.XXXXXX`
  43 trap 'exit 1' INT
  44 trap 'rm -rf "$tmpdir"' EXIT
  45
  46
  47 use_magnet_to_acquire_checksum_faster()
  48 #             ___________________
  49 #        ,--'' ~~~~~~~^^^~._     '.
  50 #    ,.-' ~~~~~~~~~~^^^^^~~~._._   \
  51 #    |   /^^^^^|    /^^^^^^^^\\ \   \
  52 #  ,/___  <  o>      <  (OO) > _     \
  53 # /'/,         |-         .       ----.\
  54 # |(|-'^^;,-  ,|     __    ^~~^^^^^^^; |\
  55 # \\`  |    <;_    __ |`---  ..-^^/- | ||
  56 #  \`-|Oq-.____`________~='^^|__,/  ' //
  57 #   \ || | |   |  |    \ ..-;|  /    '/
  58 #   | ||#|#|the|==|game!|'^` |/'    /'
  59 #   | \\\\^\***|***|    \ ,,;'     /
  60 #   |  `-=\_\__\___\__..-' ,.- - ,/
  61 #   | . `-_  ------   _,-'^-'^,-'
  62 #   | `-._________..--''^,-''^
  63 #   \             ,...-'^
  64 #    `----------'^              PROBLEM?
  65 {
  66         magnet=`GIT_DIR="$git_src_repo/.git" git ls-files -s "$1"`
  67         if [ -n "$magnet" ]; then
  68                 magnet=${magnet#* }
  69                 magnet=${magnet%% *}
  70                 echo "$magnet"
  71         else
  72                 git hash-object "$1"
  73         fi
  74 }
  75
  76 lastinfiles=
  77 lastinfileshash=
  78 acquire_checksum()
  79 {
  80         if [ x"$1/../$2" = x"$lastinfiles" ]; then
  81                 _a_s=$lastinfileshash
  82         else
  83                 _a_e=false
  84                 for _a_f in "$1" "$2"; do
  85                         case "$_a_f" in
  86                                 */background_l2.tga|*/background_ingame_l2.tga)
  87                                         _a_e=true
  88                                         ;;
  89                         esac
  90                 done
  91                 if [ -n "$git_src_repo" ] && ! $_a_e; then
  92                         _a_s=`use_magnet_to_acquire_checksum_faster "${1#./}"`
  93                         if [ -n "$2" ]; then
  94                                 _a_s=$_a_s`use_magnet_to_acquire_checksum_faster "${2#./}"`
  95                         fi
  96                 else
  97                         _a_s=`git hash-object "$1"`
  98                         if [ -n "$2" ]; then
  99                                 _a_s=$_a_s`git hash-object "$2"`
 100                         fi
 101                 fi
 102                 lastinfileshash=$_a_s
 103                 lastinfiles="$1/../$2"
 104         fi
 105         echo "$_a_s"
 106 }
 107
 108 cached()
 109 {
 110         flag=$1; shift
 111         method=$1; shift
 112         infile1=$1; shift
 113         infile2=$1; shift
 114         outfile1=$1; shift
 115         outfile2=$1; shift
 116         if ! $flag; then
 117                 return 0
 118         fi
 119         #sleep 0.25
 120         if [ x"$infile1" = x"$outfile1" ]; then
 121                 keep=true
 122         fi
 123         options=`echo "$*" | git hash-object --stdin`
 124         selfprofile convert_findchecksum
 125         sum=`acquire_checksum "$infile1" "$infile2"`
 126         selfprofile convert_makecachedir
 127         mkdir -p "$CACHEDIR/$method-$options"
 128         name1="$CACHEDIR/$method-$options/$sum-1.${outfile1##*.}"
 129         [ -z "$outfile2" ] || name2="$CACHEDIR/$method-$options/$sum-2.${outfile2##*.}"
 130         tempfile1="${name1%/*}/new-${name1##*/}"
 131         [ -z "$outfile2" ] || tempfile2="${name2%/*}/new-${name2##*/}"
 132         if [ -f "$name1" ] && { [ -z "$outfile2" ] || [ -f "$name2" ]; }; then
 133                 selfprofile convert_copyoutput
 134                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
 135                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
 136                 conv=true
 137         elif selfprofile convert_makeoutput; "$method" "$infile1" "$infile2" "$tempfile1" "$tempfile2" "$@"; then
 138                 mv "$tempfile1" "$name1"
 139                 [ -z "$outfile2" ] || mv "$tempfile2" "$name2"
 140                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
 141                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
 142                 conv=true
 143         else
 144                 selfprofile convert_cleartemp
 145                 rm -f "$tempfile1"
 146                 rm -f "$tempfile2"
 147                 selfprofile convert_finished
 148                 exit 1
 149         fi
 150         selfprofile convert_finished
 151 }
 152
 153 pickdxta()
 154 {
 155         pd_t=$1; shift
 156         pd_d=$1; shift
 157         pd_i=$1; shift
 158         pd_o=$1; shift
 159         for pd_dd in $pd_d; do
 160                 if [ -f "$pd_o" ]; then
 161                         "$meprefix"compress-texture "$pd_t" "$pd_dd" "$pd_i" "$pd_o".tmp.dds "$@"
 162                         pd_psnr_tmp=`compare -channel alpha -metric PSNR "$pd_i" "$pd_o".tmp.dds NULL: 2>&1`
 163                         case "$pd_psnr_tmp" in
 164                                 [0-9.]*)
 165                                         ;;
 166                                 *)
 167                                         pd_psnr_tmp=999.9
 168                                         ;;
 169                         esac
 170                         echo >&2 "$pd_dd: $pd_psnr_tmp dB"
 171                         pd_psnr_diff=`echo "($pd_psnr_tmp) - ($pd_psnr)" | bc -l`
 172                         case "$pd_psnr_diff" in
 173                                 -*|0)
 174                                         # tmp is smaller or equal
 175                                         # smaller PSNR is worse
 176                                         # no action
 177                                         ;;
 178                                 *)
 179                                         # tmp is larger
 180                                         # larger PSNR is better
 181                                         pd_psnr=$pd_psnr_tmp
 182                                         mv "$pd_o".tmp.dds "$pd_o"
 183                                         echo >&2 "PICKED (better)"
 184                                         ;;
 185                         esac
 186                 else
 187                         "$meprefix"compress-texture "$pd_t" "$pd_dd" "$pd_i" "$pd_o" "$@"
 188                         pd_psnr=`compare -channel alpha -metric PSNR "$pd_i" "$pd_o" NULL: 2>&1`
 189                         case "$pd_psnr" in
 190                                 [0-9.]*)
 191                                         ;;
 192                                 *)
 193                                         pd_psnr=999.9
 194                                         ;;
 195                         esac
 196                         echo >&2 "$pd_dd: $pd_psnr dB"
 197                         echo >&2 "PICKED (first)"
 198                 fi
 199         done
 200 }
 201
 202 reduce_jpeg2_dds()
 203 {
 204         i=$1; shift
 205         ia=$1; shift
 206         o=$1; shift; shift
 207         convert "$i" "$ia" -compose CopyOpacity -composite "$tmpdir/x.tga" && \
 208         pickdxta "$dds_tool" "$dds_sepalpha" "$tmpdir/x.tga" "$o" $1
 209 }
 210
 211 reduce_jpeg2_dds_premul()
 212 {
 213         i=$1; shift
 214         ia=$1; shift
 215         o=$1; shift; shift
 216         convert "$i" "$ia" -compose CopyOpacity -composite "$tmpdir/x.tga" && \
 217         pickdxta "$dds_tool" "$dds_prealpha" "$tmpdir/x.tga" "$o" $1
 218 }
 219
 220 reduce_jpeg2_jpeg2()
 221 {
 222         i=$1; shift
 223         ia=$1; shift
 224         o=$1; shift
 225         oa=$1; shift
 226         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 227                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
 228                         cp "$i" "$o"
 229                 fi
 230         else
 231                 return 1
 232         fi
 233         if convert "$ia" TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
 234                 if [ "`stat -c %s "$ia"`" -lt "`stat -c %s "$oa"`" ]; then
 235                         cp "$ia" "$oa"
 236                 fi
 237         else
 238                 return 1
 239         fi
 240 }
 241
 242 reduce_jpeg_jpeg()
 243 {
 244         i=$1; shift; shift
 245         o=$1; shift; shift
 246         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 247                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
 248                         cp "$i" "$o"
 249                 fi
 250         else
 251                 return 1
 252         fi
 253 }
 254
 255 reduce_ogg_ogg()
 256 {
 257         i=$1; shift; shift
 258         o=$1; shift; shift
 259         tags=`vorbiscomment -R -l "$i" || true`
 260         oggdec -o "$tmpdir/x.wav" "$i" && \
 261         oggenc -q"$1" -o "$o" "$tmpdir/x.wav"
 262         echo "$tags" | vorbiscomment -R -w "$o" || true
 263 }
 264
 265 reduce_wav_ogg()
 266 {
 267         i=$1; shift; shift
 268         o=$1; shift; shift
 269         oggenc -q"$1" -o "$o" "$i"
 270 }
 271
 272 reduce_rgba_dds()
 273 {
 274         i=$1; shift; shift
 275         o=$1; shift; shift
 276         convert "$i" "$tmpdir/x.tga" && \
 277         pickdxta "$dds_tool" "$dds_sepalpha" "$tmpdir/x.tga" "$o" $1
 278 }
 279
 280 reduce_rgba_dds_premul()
 281 {
 282         i=$1; shift; shift
 283         o=$1; shift; shift
 284         convert "$i" "$tmpdir/x.tga" && \
 285         pickdxta "$dds_tool" "$dds_prealpha" "$tmpdir/x.tga" "$o" $1
 286 }
 287
 288 reduce_rgba_jpeg2()
 289 {
 290         i=$1; shift; shift
 291         o=$1; shift
 292         oa=$1; shift
 293         if convert "$i" -alpha off TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 294                 :
 295         else
 296                 return 1
 297         fi
 298         if convert "$i" -alpha extract TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
 299                 :
 300         else
 301                 return 1
 302         fi
 303 }
 304
 305 reduce_rgb_dds()
 306 {
 307         i=$1; shift; shift
 308         o=$1; shift; shift
 309         convert "$i" "$tmpdir/x.tga" && \
 310         "$meprefix"compress-texture "$dds_tool" "$dds_noalpha" "$tmpdir/x.tga" "$o" $1
 311 }
 312
 313 reduce_rgb_jpeg()
 314 {
 315         i=$1; shift; shift
 316         o=$1; shift; shift
 317         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 318                 :
 319         else
 320                 return 1
 321         fi
 322 }
 323
 324 has_alpha()
 325 {
 326         i=$1; shift; shift
 327         o=$1; shift; shift
 328         if convert "$i" -depth 16 RGBA:- | perl -e 'while(read STDIN, $_, 8) { substr($_, 6, 2) eq "\xFF\xFF" or exit 1; } exit 0;'; then
 329                 # no alpha
 330                 : > "$o"
 331         else
 332                 # has alpha
 333                 echo yes > "$o"
 334         fi
 335 }
 336
 337 to_delete=
 338 for F in "$@"; do
 339         selfprofile prepareconvert
 340         f=${F%.*}
 341
 342         echo >&2 "Handling $F..."
 343         conv=false
 344         keep=false
 345         jqual_rgb=$jpeg_qual_rgb
 346         jqual_a=$jpeg_qual_a
 347
 348         will_jpeg=$do_jpeg
 349         will_dds=$do_dds
 350         will_ogg=$do_ogg
 351         if ! $ogg_ogg; then
 352                 case "$f" in
 353                         *.ogg) will_ogg=false ;;
 354                 esac
 355         fi
 356         case "$f" in
 357                 ./sounds/misc/talk*.wav) will_ogg=false ;; # engine "feature"
 358                 *_bump) will_dds=false ;;
 359                 ./models/player/*) will_dds=false ;;
 360                 ./models/sprites/*) will_dds=false ;;
 361                 ./textures/*) ;;
 362                 ./models/*) ;;
 363                 ./particles/*) ;;
 364                 ./progs/*) ;;
 365                 *)
 366                         # we can't DDS compress the 2D textures, sorry
 367                         # but JPEG is still fine
 368                         will_dds=false
 369                         ;;
 370         esac
 371
 372         # configure S2TC
 373         case "$f" in
 374                 *_norm)
 375                         export S2TC_COLORDIST_MODE=NORMALMAP
 376                         export S2TC_RANDOM_COLORS=256
 377                         export S2TC_REFINE_COLORS=LOOP
 378                         ;;
 379                 *)
 380                         export S2TC_COLORDIST_MODE=SRGB_MIXED
 381                         export S2TC_RANDOM_COLORS=64
 382                         export S2TC_REFINE_COLORS=LOOP
 383                         ;;
 384         esac
 385
 386         # for deluxemaps, lightmaps and normalmaps, enforce high jpeg quality (like on alpha channels)
 387         if [ "$jqual_a" -gt "$jqual_rgb" ]; then
 388                 case "$f" in
 389                         ./maps/*/lm_[0-9][0-9][0-9][13579]) # deluxemap
 390                                 jqual_rgb=$jqual_a
 391                                 ;;
 392                         ./maps/*/lm_[0-9][0-9][0-9][02468]) # lightmap
 393                                 jqual_rgb=$jqual_a
 394                                 ;;
 395                         *_norm) # normalmap
 396                                 jqual_rgb=$jqual_a
 397                                 ;;
 398                 esac
 399         fi
 400
 401         pm=
 402         case "$f" in
 403                 ./particles/particlefont) # particlefont uses premultiplied alpha
 404                         pm=_premul
 405                         ;;
 406         esac
 407
 408         if $do_jpeg_if_not_dds; then
 409                 if $will_dds; then
 410                         will_jpeg=false
 411                 else
 412                         will_jpeg=true
 413                 fi
 414         fi
 415         selfprofile startconvert
 416         case "$F" in
 417                 *_alpha.jpg)
 418                         # handle in *.jpg case
 419
 420                         # they always got converted, I assume
 421                         if $will_dds || $will_jpeg; then
 422                                 conv=true
 423                         fi
 424                         keep=$will_jpeg
 425                         ;;
 426                 *.jpg)
 427                         if [ -f "${f}_alpha.jpg" ]; then
 428                                 cached "$will_dds"  reduce_jpeg2_dds$pm "$F" "${f}_alpha.jpg" "dds/${f}.dds" ""               "$dds_flags"
 429                                 cached "$will_jpeg" reduce_jpeg2_jpeg2  "$F" "${f}_alpha.jpg" "$F"           "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
 430                         else
 431                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 432                                 cached "$will_jpeg" reduce_jpeg_jpeg    "$F" ""               "$F"           ""               "$jqual_rgb"
 433                         fi
 434                         ;;
 435                 *.png|*.tga)
 436                         cached true has_alpha "$F" "" "$F.hasalpha" ""
 437                         conv=false
 438                         if [ -s "$F.hasalpha" ]; then
 439                                 cached "$will_dds"  reduce_rgba_dds$pm  "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 440                                 cached "$will_jpeg" reduce_rgba_jpeg2   "$F" ""               "${f}.jpg"     "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
 441                         else
 442                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 443                                 cached "$will_jpeg" reduce_rgb_jpeg     "$F" ""               "${f}.jpg"     ""               "$jqual_rgb"
 444                         fi
 445                         rm -f "$F.hasalpha"
 446                         ;;
 447                 *.ogg)
 448                         cached "$will_ogg" reduce_ogg_ogg "$F" "" "$F" "" "$ogg_qual"
 449                         ;;
 450                 ./sound/misc/null.wav)
 451                         # never convert this one
 452                         ;;
 453                 *.wav)
 454                         cached "$will_ogg" reduce_wav_ogg "$F" "" "${f}.ogg" "" "$ogg_qual"
 455                         ;;
 456         esac
 457         selfprofile marktodelete
 458         if $del_src; then
 459                 if $conv; then
 460                         if ! $keep; then
 461                                 # FIXME can't have spaces in filenames that way
 462                                 to_delete="$to_delete $F"
 463                         fi
 464                 fi
 465         fi
 466         selfprofile symlinkfixing
 467         # fix up DDS paths by a symbolic link
 468         if [ -f "dds/${f}.dds" ]; then
 469                 if [ -z "${f##./textures/*}" ]; then
 470                         if [ -n "${f##./textures/*/*}" ]; then
 471                                 ln -snf "textures/${f#./textures/}.dds" "dds/${f#./textures/}.dds"
 472                         fi
 473                 fi
 474         fi
 475         selfprofile looping
 476 done
 477
 478 for F in $to_delete; do
 479         rm -f "$F"
 480 done
 481 selfprofile finished_time
 482 set | grep ^selfprofile_counter_ >&2