misc/tools/cached-converter.sh

   1 #!/bin/sh
   2
   3 set -e
   4
   5 : ${CACHEDIR:=$HOME/.xonotic-cached-converter}
   6 : ${do_jpeg:=true}
   7 : ${do_jpeg_if_not_dds:=false}
   8 : ${jpeg_qual_rgb:=95}
   9 : ${jpeg_qual_a:=99}
  10 : ${do_dds:=true}
  11 : ${dds_tool:=compressonator-dxtc}
  12 : ${do_ogg:=false}
  13 : ${ogg_qual:=1}
  14 : ${del_src:=false}
  15 : ${git_src_repo:=}
  16 : ${dds_noalpha:=dxt1}
  17 : ${dds_prealpha:=dxt2 dxt4}
  18 : ${dds_sepalpha:=dxt3 dxt5}
  19
  20 selfprofile_t0=`date +%s`
  21 selfprofile_step=init
  22 selfprofile()
  23 {
  24         selfprofile_t=`date +%s`
  25         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step+$selfprofile_t))"
  26         selfprofile_step=$1
  27         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step-$selfprofile_t))"
  28         selfprofile_t0=$selfprofile_t
  29 }
  30
  31 me=$0
  32 case "$me" in
  33         */*)
  34                 meprefix=${me%/*}/
  35                 ;;
  36         *)
  37                 meprefix=
  38                 ;;
  39 esac
  40
  41 tmpdir=`mktemp -d -t cached-converter.XXXXXX`
  42 trap 'exit 1' INT
  43 trap 'rm -rf "$tmpdir"' EXIT
  44
  45
  46 use_magnet_to_acquire_checksum_faster()
  47 #             ___________________
  48 #        ,--'' ~~~~~~~^^^~._     '.
  49 #    ,.-' ~~~~~~~~~~^^^^^~~~._._   \
  50 #    |   /^^^^^|    /^^^^^^^^\\ \   \
  51 #  ,/___  <  o>      <  (OO) > _     \
  52 # /'/,         |-         .       ----.\
  53 # |(|-'^^;,-  ,|     __    ^~~^^^^^^^; |\
  54 # \\`  |    <;_    __ |`---  ..-^^/- | ||
  55 #  \`-|Oq-.____`________~='^^|__,/  ' //
  56 #   \ || | |   |  |    \ ..-;|  /    '/
  57 #   | ||#|#|the|==|game!|'^` |/'    /'
  58 #   | \\\\^\***|***|    \ ,,;'     /
  59 #   |  `-=\_\__\___\__..-' ,.- - ,/
  60 #   | . `-_  ------   _,-'^-'^,-'
  61 #   | `-._________..--''^,-''^
  62 #   \             ,...-'^
  63 #    `----------'^              PROBLEM?
  64 {
  65         magnet=`GIT_DIR="$git_src_repo/.git" git ls-files -s "$1"`
  66         if [ -n "$magnet" ]; then
  67                 magnet=${magnet#* }
  68                 magnet=${magnet%% *}
  69                 echo "$magnet"
  70         else
  71                 git hash-object "$1"
  72         fi
  73 }
  74
  75 lastinfiles=
  76 lastinfileshash=
  77 acquire_checksum()
  78 {
  79         if [ x"$1/../$2" = x"$lastinfiles" ]; then
  80                 _a_s=$lastinfileshash
  81         else
  82                 _a_e=false
  83                 for _a_f in "$1" "$2"; do
  84                         case "$_a_f" in
  85                                 */background_l2.tga|*/background_ingame_l2.tga)
  86                                         _a_e=true
  87                                         ;;
  88                         esac
  89                 done
  90                 if [ -n "$git_src_repo" ] && ! $_a_e; then
  91                         _a_s=`use_magnet_to_acquire_checksum_faster "${1#./}"`
  92                         if [ -n "$2" ]; then
  93                                 _a_s=$_a_s`use_magnet_to_acquire_checksum_faster "${2#./}"`
  94                         fi
  95                 else
  96                         _a_s=`git hash-object "$1"`
  97                         if [ -n "$2" ]; then
  98                                 _a_s=$_a_s`git hash-object "$2"`
  99                         fi
 100                 fi
 101                 lastinfileshash=$_a_s
 102                 lastinfiles="$1/../$2"
 103         fi
 104         echo "$_a_s"
 105 }
 106
 107 cached()
 108 {
 109         flag=$1; shift
 110         method=$1; shift
 111         infile1=$1; shift
 112         infile2=$1; shift
 113         outfile1=$1; shift
 114         outfile2=$1; shift
 115         if ! $flag; then
 116                 return 0
 117         fi
 118         #sleep 0.25
 119         if [ x"$infile1" = x"$outfile1" ]; then
 120                 keep=true
 121         fi
 122         options=`echo "$*" | git hash-object --stdin`
 123         selfprofile convert_findchecksum
 124         sum=`acquire_checksum "$infile1" "$infile2"`
 125         selfprofile convert_makecachedir
 126         mkdir -p "$CACHEDIR/$method-$options"
 127         name1="$CACHEDIR/$method-$options/$sum-1.${outfile1##*.}"
 128         [ -z "$outfile2" ] || name2="$CACHEDIR/$method-$options/$sum-2.${outfile2##*.}"
 129         tempfile1="${name1%/*}/new-${name1##*/}"
 130         [ -z "$outfile2" ] || tempfile2="${name2%/*}/new-${name2##*/}"
 131         if [ -f "$name1" ] && { [ -z "$outfile2" ] || [ -f "$name2" ]; }; then
 132                 selfprofile convert_copyoutput
 133                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
 134                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
 135                 conv=true
 136         elif selfprofile convert_makeoutput; "$method" "$infile1" "$infile2" "$tempfile1" "$tempfile2" "$@"; then
 137                 mv "$tempfile1" "$name1"
 138                 [ -z "$outfile2" ] || mv "$tempfile2" "$name2"
 139                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
 140                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
 141                 conv=true
 142         else
 143                 selfprofile convert_cleartemp
 144                 rm -f "$tempfile1"
 145                 rm -f "$tempfile2"
 146                 selfprofile convert_finished
 147                 exit 1
 148         fi
 149         selfprofile convert_finished
 150 }
 151
 152 pickdxta()
 153 {
 154         pd_t=$1; shift
 155         pd_d=$1; shift
 156         pd_i=$1; shift
 157         pd_o=$1; shift
 158         for pd_dd in $pd_d; do
 159                 if [ -f "$pd_o" ]; then
 160                         "$meprefix"compress-texture "$pd_t" "$pd_dd" "$pd_i" "$pd_o".tmp.dds "$@"
 161                         pd_psnr_tmp=`compare -channel alpha -metric PSNR "$pd_i" "$pd_o".tmp.dds NULL: 2>&1`
 162                         case "$pd_psnr_tmp" in
 163                                 [0-9.]*)
 164                                         ;;
 165                                 *)
 166                                         pd_psnr_tmp=999.9
 167                                         ;;
 168                         esac
 169                         echo >&2 "$pd_dd: $pd_psnr_tmp dB"
 170                         pd_psnr_diff=`echo "($pd_psnr_tmp) - ($pd_psnr)" | bc -l`
 171                         case "$pd_psnr_diff" in
 172                                 -*|0)
 173                                         # tmp is smaller or equal
 174                                         # smaller PSNR is worse
 175                                         # no action
 176                                         ;;
 177                                 *)
 178                                         # tmp is larger
 179                                         # larger PSNR is better
 180                                         pd_psnr=$pd_psnr_tmp
 181                                         mv "$pd_o".tmp.dds "$pd_o"
 182                                         echo >&2 "PICKED (better)"
 183                                         ;;
 184                         esac
 185                 else
 186                         "$meprefix"compress-texture "$pd_t" "$pd_dd" "$pd_i" "$pd_o" "$@"
 187                         pd_psnr=`compare -channel alpha -metric PSNR "$pd_i" "$pd_o" NULL: 2>&1`
 188                         case "$pd_psnr" in
 189                                 [0-9.]*)
 190                                         ;;
 191                                 *)
 192                                         pd_psnr=999.9
 193                                         ;;
 194                         esac
 195                         echo >&2 "$pd_dd: $pd_psnr dB"
 196                         echo >&2 "PICKED (first)"
 197                 fi
 198         done
 199 }
 200
 201 reduce_jpeg2_dds()
 202 {
 203         i=$1; shift
 204         ia=$1; shift
 205         o=$1; shift; shift
 206         convert "$i" "$ia" -compose CopyOpacity -composite "$tmpdir/x.tga" && \
 207         pickdxta "$dds_tool" "$dds_sepalpha" "$tmpdir/x.tga" "$o" $1
 208 }
 209
 210 reduce_jpeg2_dds_premul()
 211 {
 212         i=$1; shift
 213         ia=$1; shift
 214         o=$1; shift; shift
 215         convert "$i" "$ia" -compose CopyOpacity -composite "$tmpdir/x.tga" && \
 216         pickdxta "$dds_tool" "$dds_prealpha" "$tmpdir/x.tga" "$o" $1
 217 }
 218
 219 reduce_jpeg2_jpeg2()
 220 {
 221         i=$1; shift
 222         ia=$1; shift
 223         o=$1; shift
 224         oa=$1; shift
 225         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 226                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
 227                         cp "$i" "$o"
 228                 fi
 229         else
 230                 return 1
 231         fi
 232         if convert "$ia" TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
 233                 if [ "`stat -c %s "$ia"`" -lt "`stat -c %s "$oa"`" ]; then
 234                         cp "$ia" "$oa"
 235                 fi
 236         else
 237                 return 1
 238         fi
 239 }
 240
 241 reduce_jpeg_jpeg()
 242 {
 243         i=$1; shift; shift
 244         o=$1; shift; shift
 245         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 246                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
 247                         cp "$i" "$o"
 248                 fi
 249         else
 250                 return 1
 251         fi
 252 }
 253
 254 reduce_ogg_ogg()
 255 {
 256         i=$1; shift; shift
 257         o=$1; shift; shift
 258         tags=`vorbiscomment -R -l "$i" || true`
 259         oggdec -o "$tmpdir/x.wav" "$i" && \
 260         oggenc -q"$1" -o "$o" "$tmpdir/x.wav"
 261         echo "$tags" | vorbiscomment -R -w "$o" || true
 262 }
 263
 264 reduce_wav_ogg()
 265 {
 266         i=$1; shift; shift
 267         o=$1; shift; shift
 268         oggenc -q"$1" -o "$o" "$i"
 269 }
 270
 271 reduce_rgba_dds()
 272 {
 273         i=$1; shift; shift
 274         o=$1; shift; shift
 275         convert "$i" "$tmpdir/x.tga" && \
 276         pickdxta "$dds_tool" "$dds_sepalpha" "$tmpdir/x.tga" "$o" $1
 277 }
 278
 279 reduce_rgba_dds_premul()
 280 {
 281         i=$1; shift; shift
 282         o=$1; shift; shift
 283         convert "$i" "$tmpdir/x.tga" && \
 284         pickdxta "$dds_tool" "$dds_prealpha" "$tmpdir/x.tga" "$o" $1
 285 }
 286
 287 reduce_rgba_jpeg2()
 288 {
 289         i=$1; shift; shift
 290         o=$1; shift
 291         oa=$1; shift
 292         if convert "$i" -alpha off TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 293                 :
 294         else
 295                 return 1
 296         fi
 297         if convert "$i" -alpha extract TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
 298                 :
 299         else
 300                 return 1
 301         fi
 302 }
 303
 304 reduce_rgb_dds()
 305 {
 306         i=$1; shift; shift
 307         o=$1; shift; shift
 308         convert "$i" "$tmpdir/x.tga" && \
 309         "$meprefix"compress-texture "$dds_tool" "$dds_noalpha" "$tmpdir/x.tga" "$o" $1
 310 }
 311
 312 reduce_rgb_jpeg()
 313 {
 314         i=$1; shift; shift
 315         o=$1; shift; shift
 316         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 317                 :
 318         else
 319                 return 1
 320         fi
 321 }
 322
 323 has_alpha()
 324 {
 325         i=$1; shift; shift
 326         o=$1; shift; shift
 327         if convert "$i" -depth 16 RGBA:- | perl -e 'while(read STDIN, $_, 8) { substr($_, 6, 2) eq "\xFF\xFF" or exit 1; } exit 0;'; then
 328                 # no alpha
 329                 : > "$o"
 330         else
 331                 # has alpha
 332                 echo yes > "$o"
 333         fi
 334 }
 335
 336 to_delete=
 337 for F in "$@"; do
 338         selfprofile prepareconvert
 339         f=${F%.*}
 340
 341         echo >&2 "Handling $F..."
 342         conv=false
 343         keep=false
 344         jqual_rgb=$jpeg_qual_rgb
 345         jqual_a=$jpeg_qual_a
 346
 347         will_jpeg=$do_jpeg
 348         will_dds=$do_dds
 349         will_ogg=$do_ogg
 350         case "$f" in
 351                 ./sounds/misc/talk*.wav) will_ogg=false ;; # engine "feature"
 352                 *_bump) will_dds=false ;;
 353                 ./models/player/*) will_dds=false ;;
 354                 ./models/sprites/*) will_dds=false ;;
 355                 ./textures/*) ;;
 356                 ./models/*) ;;
 357                 ./particles/*) ;;
 358                 ./progs/*) ;;
 359                 *)
 360                         # we can't DDS compress the 2D textures, sorry
 361                         # but JPEG is still fine
 362                         will_dds=false
 363                         ;;
 364         esac
 365
 366         # configure S2TC
 367         case "$f" in
 368                 *_norm)
 369                         export S2TC_COLORDIST_MODE=NORMALMAP
 370                         export S2TC_RANDOM_COLORS=256
 371                         export S2TC_REFINE_COLORS=LOOP
 372                         ;;
 373                 *)
 374                         export S2TC_COLORDIST_MODE=SRGB_MIXED
 375                         export S2TC_RANDOM_COLORS=64
 376                         export S2TC_REFINE_COLORS=LOOP
 377                         ;;
 378         esac
 379
 380         # for deluxemaps, lightmaps and normalmaps, enforce high jpeg quality (like on alpha channels)
 381         if [ "$jqual_a" -gt "$jqual_rgb" ]; then
 382                 case "$f" in
 383                         ./maps/*/lm_[0-9][0-9][0-9][13579]) # deluxemap
 384                                 jqual_rgb=$jqual_a
 385                                 ;;
 386                         ./maps/*/lm_[0-9][0-9][0-9][02468]) # lightmap
 387                                 jqual_rgb=$jqual_a
 388                                 ;;
 389                         *_norm) # normalmap
 390                                 jqual_rgb=$jqual_a
 391                                 ;;
 392                 esac
 393         fi
 394
 395         pm=
 396         case "$f" in
 397                 ./particles/particlefont) # particlefont uses premultiplied alpha
 398                         pm=_premul
 399                         ;;
 400         esac
 401
 402         if $do_jpeg_if_not_dds; then
 403                 if $will_dds; then
 404                         will_jpeg=false
 405                 else
 406                         will_jpeg=true
 407                 fi
 408         fi
 409         selfprofile startconvert
 410         case "$F" in
 411                 *_alpha.jpg)
 412                         # handle in *.jpg case
 413
 414                         # they always got converted, I assume
 415                         if $will_dds || $will_jpeg; then
 416                                 conv=true
 417                         fi
 418                         keep=$will_jpeg
 419                         ;;
 420                 *.jpg)
 421                         if [ -f "${f}_alpha.jpg" ]; then
 422                                 cached "$will_dds"  reduce_jpeg2_dds$pm "$F" "${f}_alpha.jpg" "dds/${f}.dds" ""               "$dds_flags"
 423                                 cached "$will_jpeg" reduce_jpeg2_jpeg2  "$F" "${f}_alpha.jpg" "$F"           "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
 424                         else
 425                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 426                                 cached "$will_jpeg" reduce_jpeg_jpeg    "$F" ""               "$F"           ""               "$jqual_rgb"
 427                         fi
 428                         ;;
 429                 *.png|*.tga)
 430                         cached true has_alpha "$F" "" "$F.hasalpha" ""
 431                         conv=false
 432                         if [ -s "$F.hasalpha" ]; then
 433                                 cached "$will_dds"  reduce_rgba_dds$pm  "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 434                                 cached "$will_jpeg" reduce_rgba_jpeg2   "$F" ""               "${f}.jpg"     "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
 435                         else
 436                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 437                                 cached "$will_jpeg" reduce_rgb_jpeg     "$F" ""               "${f}.jpg"     ""               "$jqual_rgb"
 438                         fi
 439                         rm -f "$F.hasalpha"
 440                         ;;
 441                 *.ogg)
 442                         cached "$will_ogg" reduce_ogg_ogg "$F" "" "$F" "" "$ogg_qual"
 443                         ;;
 444                 ./sound/misc/null.wav)
 445                         # never convert this one
 446                         ;;
 447                 *.wav)
 448                         cached "$will_ogg" reduce_wav_ogg "$F" "" "${f}.ogg" "" "$ogg_qual"
 449                         ;;
 450         esac
 451         selfprofile marktodelete
 452         if $del_src; then
 453                 if $conv; then
 454                         if ! $keep; then
 455                                 # FIXME can't have spaces in filenames that way
 456                                 to_delete="$to_delete $F"
 457                         fi
 458                 fi
 459         fi
 460         selfprofile symlinkfixing
 461         # fix up DDS paths by a symbolic link
 462         if [ -f "dds/${f}.dds" ]; then
 463                 if [ -z "${f##./textures/*}" ]; then
 464                         if [ -n "${f##./textures/*/*}" ]; then
 465                                 ln -snf "textures/${f#./textures/}.dds" "dds/${f#./textures/}.dds"
 466                         fi
 467                 fi
 468         fi
 469         selfprofile looping
 470 done
 471
 472 for F in $to_delete; do
 473         rm -f "$F"
 474 done
 475 selfprofile finished_time
 476 set | grep ^selfprofile_counter_ >&2