misc/tools/cached-converter.sh

   1 #!/bin/sh
   2
   3 set -e
   4
   5 : ${CACHEDIR:=$HOME/.xonotic-cached-converter}
   6 : ${do_jpeg:=true}
   7 : ${do_jpeg_if_not_dds:=false}
   8 : ${jpeg_qual_rgb:=95}
   9 : ${jpeg_qual_a:=99}
  10 : ${do_dds:=true}
  11 : ${dds_tool:=compressonator-dxtc}
  12 : ${do_ogg:=false}
  13 : ${ogg_qual:=1}
  14 : ${del_src:=false}
  15 : ${git_src_repo:=}
  16
  17 selfprofile_t0=`date +%s`
  18 selfprofile_step=init
  19 selfprofile()
  20 {
  21         selfprofile_t=`date +%s`
  22         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step+$selfprofile_t))"
  23         selfprofile_step=$1
  24         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step-$selfprofile_t))"
  25         selfprofile_t0=$selfprofile_t
  26 }
  27
  28 me=$0
  29 case "$me" in
  30         */*)
  31                 meprefix=${me%/*}/
  32                 ;;
  33         *)
  34                 meprefix=
  35                 ;;
  36 esac
  37
  38 tmpdir=`mktemp -d -t cached-converter.XXXXXX`
  39 trap 'exit 1' INT
  40 trap 'rm -rf "$tmpdir"' EXIT
  41
  42
  43 use_magnet_to_acquire_checksum_faster()
  44 #             ___________________
  45 #        ,--'' ~~~~~~~^^^~._     '.
  46 #    ,.-' ~~~~~~~~~~^^^^^~~~._._   \
  47 #    |   /^^^^^|    /^^^^^^^^\\ \   \
  48 #  ,/___  <  o>      <  (OO) > _     \
  49 # /'/,         |-         .       ----.\
  50 # |(|-'^^;,-  ,|     __    ^~~^^^^^^^; |\
  51 # \\`  |    <;_    __ |`---  ..-^^/- | ||
  52 #  \`-|Oq-.____`________~='^^|__,/  ' //
  53 #   \ || | |   |  |    \ ..-;|  /    '/
  54 #   | ||#|#|the|==|game!|'^` |/'    /'
  55 #   | \\\\^\***|***|    \ ,,;'     /
  56 #   |  `-=\_\__\___\__..-' ,.- - ,/
  57 #   | . `-_  ------   _,-'^-'^,-'
  58 #   | `-._________..--''^,-''^
  59 #   \             ,...-'^
  60 #    `----------'^              PROBLEM?
  61 {
  62         magnet=`GIT_DIR="$git_src_repo/.git" git ls-files -s "$1"`
  63         if [ -n "$magnet" ]; then
  64                 magnet=${magnet#* }
  65                 magnet=${magnet%% *}
  66                 sum=$sum$magnet
  67         else
  68                 sum=$sum`git hash-object "$1"`
  69         fi
  70 }
  71
  72 lastinfiles=
  73 lastinfileshash=
  74 cached()
  75 {
  76         flag=$1; shift
  77         method=$1; shift
  78         infile1=$1; shift
  79         infile2=$1; shift
  80         outfile1=$1; shift
  81         outfile2=$1; shift
  82         if ! $flag; then
  83                 return 0
  84         fi
  85         #sleep 0.25
  86         if [ x"$infile1" = x"$outfile1" ]; then
  87                 keep=true
  88         fi
  89         options=`echo "$*" | git hash-object --stdin`
  90         selfprofile convert_findchecksum
  91         if [ x"$infile1/../$infile2" = x"$lastinfiles" ]; then
  92                 sum=$lastinfileshash
  93         else
  94                 evil=false
  95                 for infile in "$infile1" "$infile2"; do
  96                         case "$infile" in
  97                                 */background_l2.tga|*/background_ingame_l2.tga)
  98                                         evil=true
  99                                         ;;
 100                         esac
 101                 done
 102                 if [ -n "$git_src_repo" ] && ! $evil; then
 103                         sum=
 104                         use_magnet_to_acquire_checksum_faster "${infile1#./}"
 105                         if [ -n "$infile2" ]; then
 106                                 use_magnet_to_acquire_checksum_faster "${infile2#./}"
 107                         fi
 108                 else
 109                         sum=`git hash-object "$infile1"`
 110                         if [ -n "$infile2" ]; then
 111                                 sum=$sum`git hash-object "$infile2"`
 112                         fi
 113                 fi
 114                 lastinfileshash=$sum
 115         fi
 116         selfprofile convert_makecachedir
 117         mkdir -p "$CACHEDIR/$method-$options"
 118         name1="$CACHEDIR/$method-$options/$sum-1.${outfile1##*.}"
 119         [ -z "$outfile2" ] || name2="$CACHEDIR/$method-$options/$sum-2.${outfile2##*.}"
 120         tempfile1="${name1%/*}/new-${name1##*/}"
 121         [ -z "$outfile2" ] || tempfile2="${name2%/*}/new-${name2##*/}"
 122         if [ -f "$name1" ] && { [ -z "$outfile2" ] || [ -f "$name2" ]; }; then
 123                 selfprofile convert_copyoutput
 124                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
 125                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
 126                 conv=true
 127         elif selfprofile convert_makeoutput; "$method" "$infile1" "$infile2" "$tempfile1" "$tempfile2" "$@"; then
 128                 mv "$tempfile1" "$name1"
 129                 [ -z "$outfile2" ] || mv "$tempfile2" "$name2"
 130                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
 131                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
 132                 conv=true
 133         else
 134                 selfprofile convert_cleartemp
 135                 rm -f "$tempfile1"
 136                 rm -f "$tempfile2"
 137                 selfprofile convert_finished
 138                 exit 1
 139         fi
 140         selfprofile convert_finished
 141 }
 142
 143 reduce_jpeg2_dds()
 144 {
 145         i=$1; shift
 146         ia=$1; shift
 147         o=$1; shift; shift
 148         convert "$i" "$ia" -compose CopyOpacity -composite "$tmpdir/x.tga" && \
 149         "$meprefix"compress-texture "$dds_tool" dxt5 "$tmpdir/x.tga" "$o" $1
 150 }
 151
 152 reduce_jpeg2_dds_premul()
 153 {
 154         i=$1; shift
 155         ia=$1; shift
 156         o=$1; shift; shift
 157         convert "$i" "$ia" -compose CopyOpacity -composite "$tmpdir/x.tga" && \
 158         "$meprefix"compress-texture "$dds_tool" dxt4 "$tmpdir/x.tga" "$o" $1
 159 }
 160
 161 reduce_jpeg2_jpeg2()
 162 {
 163         i=$1; shift
 164         ia=$1; shift
 165         o=$1; shift
 166         oa=$1; shift
 167         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 168                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
 169                         cp "$i" "$o"
 170                 fi
 171         else
 172                 return 1
 173         fi
 174         if convert "$ia" TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
 175                 if [ "`stat -c %s "$ia"`" -lt "`stat -c %s "$oa"`" ]; then
 176                         cp "$ia" "$oa"
 177                 fi
 178         else
 179                 return 1
 180         fi
 181 }
 182
 183 reduce_jpeg_jpeg()
 184 {
 185         i=$1; shift; shift
 186         o=$1; shift; shift
 187         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 188                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
 189                         cp "$i" "$o"
 190                 fi
 191         else
 192                 return 1
 193         fi
 194 }
 195
 196 reduce_ogg_ogg()
 197 {
 198         i=$1; shift; shift
 199         o=$1; shift; shift
 200         tags=`vorbiscomment -R -l "$i" || true`
 201         oggdec -o "$tmpdir/x.wav" "$i" && \
 202         oggenc -q"$1" -o "$o" "$tmpdir/x.wav"
 203         echo "$tags" | vorbiscomment -R -w "$o" || true
 204 }
 205
 206 reduce_wav_ogg()
 207 {
 208         i=$1; shift; shift
 209         o=$1; shift; shift
 210         oggenc -q"$1" -o "$o" "$i"
 211 }
 212
 213 reduce_rgba_dds()
 214 {
 215         i=$1; shift; shift
 216         o=$1; shift; shift
 217         convert "$i" "$tmpdir/x.tga" && \
 218         "$meprefix"compress-texture "$dds_tool" dxt5 "$tmpdir/x.tga" "$o" $1
 219 }
 220
 221 reduce_rgba_dds_premul()
 222 {
 223         i=$1; shift; shift
 224         o=$1; shift; shift
 225         convert "$i" "$tmpdir/x.tga" && \
 226         "$meprefix"compress-texture "$dds_tool" dxt4 "$tmpdir/x.tga" "$o" $1
 227 }
 228
 229 reduce_rgba_jpeg2()
 230 {
 231         i=$1; shift; shift
 232         o=$1; shift
 233         oa=$1; shift
 234         if convert "$i" -alpha off TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 235                 :
 236         else
 237                 return 1
 238         fi
 239         if convert "$i" -alpha extract TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
 240                 :
 241         else
 242                 return 1
 243         fi
 244 }
 245
 246 reduce_rgb_dds()
 247 {
 248         i=$1; shift; shift
 249         o=$1; shift; shift
 250         convert "$i" "$tmpdir/x.tga" && \
 251         "$meprefix"compress-texture "$dds_tool" dxt1 "$tmpdir/x.tga" "$o" $1
 252 }
 253
 254 reduce_rgb_jpeg()
 255 {
 256         i=$1; shift; shift
 257         o=$1; shift; shift
 258         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 259                 :
 260         else
 261                 return 1
 262         fi
 263 }
 264
 265 has_alpha()
 266 {
 267         i=$1; shift; shift
 268         o=$1; shift; shift
 269         if convert "$i" -depth 16 RGBA:- | perl -e 'while(read STDIN, $_, 8) { substr($_, 6, 2) eq "\xFF\xFF" or exit 1; } exit 0;'; then
 270                 # no alpha
 271                 : > "$o"
 272         else
 273                 # has alpha
 274                 echo yes > "$o"
 275         fi
 276 }
 277
 278 to_delete=
 279 for F in "$@"; do
 280         selfprofile prepareconvert
 281         f=${F%.*}
 282
 283         echo >&2 "Handling $F..."
 284         conv=false
 285         keep=false
 286         jqual_rgb=$jpeg_qual_rgb
 287         jqual_a=$jpeg_qual_a
 288
 289         will_jpeg=$do_jpeg
 290         will_dds=$do_dds
 291         case "$f" in
 292                 *_bump) will_dds=false ;;
 293                 ./models/player/*) will_dds=false ;;
 294                 ./models/sprites/*) will_dds=false ;;
 295                 ./textures/*) ;;
 296                 ./models/*) ;;
 297                 ./particles/*) ;;
 298                 ./progs/*) ;;
 299                 *)
 300                         # we can't DDS compress the 2D textures, sorry
 301                         # but JPEG is still fine
 302                         will_dds=false
 303                         ;;
 304         esac
 305
 306         # for deluxemaps, lightmaps and normalmaps, enforce high jpeg quality (like on alpha channels)
 307         if [ "$jqual_a" -gt "$jqual_rgb" ]; then
 308                 case "$f" in
 309                         ./maps/*/lm_[0-9][0-9][0-9][13579]) # deluxemap
 310                                 jqual_rgb=$jqual_a
 311                                 ;;
 312                         ./maps/*/lm_[0-9][0-9][0-9][02468]) # lightmap
 313                                 jqual_rgb=$jqual_a
 314                                 ;;
 315                         *_norm) # normalmap
 316                                 jqual_rgb=$jqual_a
 317                                 ;;
 318                 esac
 319         fi
 320
 321         pm=
 322         case "$f" in
 323                 ./particles/particlefont) # particlefont uses premultiplied alpha
 324                         pm=_premul
 325                         ;;
 326         esac
 327
 328         if $do_jpeg_if_not_dds; then
 329                 if $will_dds; then
 330                         will_jpeg=false
 331                 else
 332                         will_jpeg=true
 333                 fi
 334         fi
 335         selfprofile startconvert
 336         case "$F" in
 337                 *_alpha.jpg)
 338                         # handle in *.jpg case
 339
 340                         # they always got converted, I assume
 341                         if $will_dds || $will_jpeg; then
 342                                 conv=true
 343                         fi
 344                         keep=$will_jpeg
 345                         ;;
 346                 *.jpg)
 347                         if [ -f "${f}_alpha.jpg" ]; then
 348                                 cached "$will_dds"  reduce_jpeg2_dds$pm "$F" "${f}_alpha.jpg" "dds/${f}.dds" ""               "$dds_flags"
 349                                 cached "$will_jpeg" reduce_jpeg2_jpeg2  "$F" "${f}_alpha.jpg" "$F"           "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
 350                         else
 351                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 352                                 cached "$will_jpeg" reduce_jpeg_jpeg    "$F" ""               "$F"           ""               "$jqual_rgb"
 353                         fi
 354                         ;;
 355                 *.png|*.tga)
 356                         cached true has_alpha "$F" "" "$F.hasalpha" ""
 357                         conv=false
 358                         if [ -s "$F.hasalpha" ]; then
 359                                 cached "$will_dds"  reduce_rgba_dds$pm  "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 360                                 cached "$will_jpeg" reduce_rgba_jpeg2   "$F" ""               "${f}.jpg"     "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
 361                         else
 362                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 363                                 cached "$will_jpeg" reduce_rgb_jpeg     "$F" ""               "${f}.jpg"     ""               "$jqual_rgb"
 364                         fi
 365                         rm -f "$F.hasalpha"
 366                         ;;
 367                 *.ogg)
 368                         cached "$do_ogg" reduce_ogg_ogg "$F" "" "$F" "" "$ogg_qual"
 369                         ;;
 370                 ./sound/misc/null.wav)
 371                         # never convert this one
 372                         ;;
 373                 *.wav)
 374                         cached "$do_ogg" reduce_wav_ogg "$F" "" "${f}.ogg" "" "$ogg_qual"
 375                         ;;
 376         esac
 377         selfprofile marktodelete
 378         if $del_src; then
 379                 if $conv; then
 380                         if ! $keep; then
 381                                 # FIXME can't have spaces in filenames that way
 382                                 to_delete="$to_delete $F"
 383                         fi
 384                 fi
 385         fi
 386         selfprofile symlinkfixing
 387         # fix up DDS paths by a symbolic link
 388         if [ -f "dds/${f}.dds" ]; then
 389                 if [ -z "${f##./textures/*}" ]; then
 390                         if [ -n "${f##./textures/*/*}" ]; then
 391                                 ln -snf "textures/${f#./textures/}.dds" "dds/${f#./textures/}.dds"
 392                         fi
 393                 fi
 394         fi
 395         selfprofile looping
 396 done
 397
 398 for F in $to_delete; do
 399         rm -f "$F"
 400 done
 401 selfprofile finished_time
 402 set | grep ^selfprofile_counter_ >&2