misc/tools/cached-converter.sh

   1 #!/bin/sh
   2
   3 set -e
   4
   5 : ${CACHEDIR:=$HOME/.xonotic-cached-converter}
   6 : ${do_jpeg:=true}
   7 : ${do_jpeg_if_not_dds:=false}
   8 : ${jpeg_qual_rgb:=95}
   9 : ${jpeg_qual_a:=99}
  10 : ${do_dds:=true}
  11 : ${dds_tool:=compressonator-dxtc}
  12 : ${do_ogg:=false}
  13 : ${ogg_qual:=1}
  14 : ${del_src:=false}
  15 : ${git_src_repo:=}
  16
  17 selfprofile_t0=`date +%s`
  18 selfprofile_step=init
  19 selfprofile()
  20 {
  21         selfprofile_t=`date +%s`
  22         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step+$selfprofile_t))"
  23         selfprofile_step=$1
  24         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step-$selfprofile_t))"
  25         selfprofile_t0=$selfprofile_t
  26 }
  27
  28 me=$0
  29 case "$me" in
  30         */*)
  31                 meprefix=${me%/*}/
  32                 ;;
  33         *)
  34                 meprefix=
  35                 ;;
  36 esac
  37
  38 tmpdir=`mktemp -d -t cached-converter.XXXXXX`
  39 trap 'exit 1' INT
  40 trap 'rm -rf "$tmpdir"' EXIT
  41
  42
  43 use_magnet_to_acquire_checksum_faster()
  44 #             ___________________
  45 #        ,--'' ~~~~~~~^^^~._     '.
  46 #    ,.-' ~~~~~~~~~~^^^^^~~~._._   \
  47 #    |   /^^^^^|    /^^^^^^^^\\ \   \
  48 #  ,/___  <  o>      <  (OO) > _     \
  49 # /'/,         |-         .       ----.\
  50 # |(|-'^^;,-  ,|     __    ^~~^^^^^^^; |\
  51 # \\`  |    <;_    __ |`---  ..-^^/- | ||
  52 #  \`-|Oq-.____`________~='^^|__,/  ' //
  53 #   \ || | |   |  |    \ ..-;|  /    '/
  54 #   | ||#|#|the|==|game!|'^` |/'    /'
  55 #   | \\\\^\***|***|    \ ,,;'     /
  56 #   |  `-=\_\__\___\__..-' ,.- - ,/
  57 #   | . `-_  ------   _,-'^-'^,-'
  58 #   | `-._________..--''^,-''^
  59 #   \             ,...-'^
  60 #    `----------'^              PROBLEM?
  61 {
  62         magnet=`GIT_DIR="$git_src_repo/.git" git ls-files -s "$1"`
  63         if [ -n "$magnet" ]; then
  64                 magnet=${magnet#* }
  65                 magnet=${magnet%% *}
  66                 sum=$sum$magnet
  67         else
  68                 sum=$sum`git hash-object "$1"`
  69         fi
  70 }
  71
  72 lastinfiles=
  73 lastinfileshash=
  74 cached()
  75 {
  76         flag=$1; shift
  77         method=$1; shift
  78         infile1=$1; shift
  79         infile2=$1; shift
  80         outfile1=$1; shift
  81         outfile2=$1; shift
  82         if ! $flag; then
  83                 return 0
  84         fi
  85         #sleep 0.25
  86         if [ x"$infile1" = x"$outfile1" ]; then
  87                 keep=true
  88         fi
  89         options=`echo "$*" | git hash-object --stdin`
  90         selfprofile convert_findchecksum
  91         if [ x"$infile1/../$infile2" = x"$lastinfiles" ]; then
  92                 sum=$lastinfileshash
  93         else
  94                 evil=false
  95                 for infile in "$infile1" "$infile2"; do
  96                         case "$infile" in
  97                                 */background_l2.tga|*/background_ingame_l2.tga)
  98                                         evil=true
  99                                         ;;
 100                         esac
 101                 done
 102                 if [ -n "$git_src_repo" ] && ! $evil; then
 103                         sum=
 104                         use_magnet_to_acquire_checksum_faster "${infile1#./}"
 105                         if [ -n "$infile2" ]; then
 106                                 use_magnet_to_acquire_checksum_faster "${infile2#./}"
 107                         fi
 108                 else
 109                         sum=`git hash-object "$infile1"`
 110                         if [ -n "$infile2" ]; then
 111                                 sum=$sum`git hash-object "$infile2"`
 112                         fi
 113                 fi
 114                 lastinfileshash=$sum
 115         fi
 116         selfprofile convert_makecachedir
 117         mkdir -p "$CACHEDIR/$method-$options"
 118         name1="$CACHEDIR/$method-$options/$sum-1.${outfile1##*.}"
 119         [ -z "$outfile2" ] || name2="$CACHEDIR/$method-$options/$sum-2.${outfile2##*.}"
 120         tempfile1="${name1%/*}/new-${name1##*/}"
 121         [ -z "$outfile2" ] || tempfile2="${name2%/*}/new-${name2##*/}"
 122         if [ -f "$name1" ] && { [ -z "$outfile2" ] || [ -f "$name2" ]; }; then
 123                 selfprofile convert_copyoutput
 124                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
 125                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
 126                 conv=true
 127         elif selfprofile convert_makeoutput; "$method" "$infile1" "$infile2" "$tempfile1" "$tempfile2" "$@"; then
 128                 mv "$tempfile1" "$name1"
 129                 [ -z "$outfile2" ] || mv "$tempfile2" "$name2"
 130                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
 131                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
 132                 conv=true
 133         else
 134                 selfprofile convert_cleartemp
 135                 rm -f "$tempfile1"
 136                 rm -f "$tempfile2"
 137                 selfprofile convert_finished
 138                 exit 1
 139         fi
 140         selfprofile convert_finished
 141 }
 142
 143 reduce_jpeg2_dds()
 144 {
 145         i=$1; shift
 146         ia=$1; shift
 147         o=$1; shift; shift
 148         convert "$i" "$ia" -compose CopyOpacity -composite "$tmpdir/x.tga" && \
 149         "$meprefix"compress-texture "$dds_tool" dxt5 "$tmpdir/x.tga" "$o" $1
 150 }
 151
 152 reduce_jpeg2_dds_premul()
 153 {
 154         i=$1; shift
 155         ia=$1; shift
 156         o=$1; shift; shift
 157         convert "$i" "$ia" -compose CopyOpacity -composite "$tmpdir/x.tga" && \
 158         "$meprefix"compress-texture "$dds_tool" dxt4 "$tmpdir/x.tga" "$o" $1
 159 }
 160
 161 reduce_jpeg2_jpeg2()
 162 {
 163         i=$1; shift
 164         ia=$1; shift
 165         o=$1; shift
 166         oa=$1; shift
 167         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 168                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
 169                         cp "$i" "$o"
 170                 fi
 171         else
 172                 return 1
 173         fi
 174         if convert "$ia" TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
 175                 if [ "`stat -c %s "$ia"`" -lt "`stat -c %s "$oa"`" ]; then
 176                         cp "$ia" "$oa"
 177                 fi
 178         else
 179                 return 1
 180         fi
 181 }
 182
 183 reduce_jpeg_jpeg()
 184 {
 185         i=$1; shift; shift
 186         o=$1; shift; shift
 187         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 188                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
 189                         cp "$i" "$o"
 190                 fi
 191         else
 192                 return 1
 193         fi
 194 }
 195
 196 reduce_ogg_ogg()
 197 {
 198         i=$1; shift; shift
 199         o=$1; shift; shift
 200         tags=`vorbiscomment -R -l "$i" || true`
 201         oggdec -o "$tmpdir/x.wav" "$i" && \
 202         oggenc -q"$1" -o "$o" "$tmpdir/x.wav"
 203         echo "$tags" | vorbiscomment -R -w "$o" || true
 204 }
 205
 206 reduce_wav_ogg()
 207 {
 208         i=$1; shift; shift
 209         o=$1; shift; shift
 210         oggenc -q"$1" -o "$o" "$i"
 211 }
 212
 213 reduce_rgba_dds()
 214 {
 215         i=$1; shift; shift
 216         o=$1; shift; shift
 217         convert "$i" "$tmpdir/x.tga" && \
 218         "$meprefix"compress-texture "$dds_tool" dxt5 "$tmpdir/x.tga" "$o" $1
 219 }
 220
 221 reduce_rgba_dds_premul()
 222 {
 223         i=$1; shift; shift
 224         o=$1; shift; shift
 225         convert "$i" "$tmpdir/x.tga" && \
 226         "$meprefix"compress-texture "$dds_tool" dxt4 "$tmpdir/x.tga" "$o" $1
 227 }
 228
 229 reduce_rgba_jpeg2()
 230 {
 231         i=$1; shift; shift
 232         o=$1; shift
 233         oa=$1; shift
 234         if convert "$i" -alpha off TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 235                 :
 236         else
 237                 return 1
 238         fi
 239         if convert "$i" -alpha extract TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
 240                 :
 241         else
 242                 return 1
 243         fi
 244 }
 245
 246 reduce_rgb_dds()
 247 {
 248         i=$1; shift; shift
 249         o=$1; shift; shift
 250         convert "$i" "$tmpdir/x.tga" && \
 251         "$meprefix"compress-texture "$dds_tool" dxt1 "$tmpdir/x.tga" "$o" $1
 252 }
 253
 254 reduce_rgb_jpeg()
 255 {
 256         i=$1; shift; shift
 257         o=$1; shift; shift
 258         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 259                 :
 260         else
 261                 return 1
 262         fi
 263 }
 264
 265 has_alpha()
 266 {
 267         i=$1; shift; shift
 268         o=$1; shift; shift
 269         if convert "$i" -depth 16 RGBA:- | perl -e 'while(read STDIN, $_, 8) { substr($_, 6, 2) eq "\xFF\xFF" or exit 1; } exit 0;'; then
 270                 # no alpha
 271                 : > "$o"
 272         else
 273                 # has alpha
 274                 echo yes > "$o"
 275         fi
 276 }
 277
 278 to_delete=
 279 for F in "$@"; do
 280         selfprofile prepareconvert
 281         f=${F%.*}
 282
 283         echo >&2 "Handling $F..."
 284         conv=false
 285         keep=false
 286         jqual_rgb=$jpeg_qual_rgb
 287         jqual_a=$jpeg_qual_a
 288
 289         will_jpeg=$do_jpeg
 290         will_dds=$do_dds
 291         will_ogg=$do_ogg
 292         case "$f" in
 293                 ./sounds/misc/talk*.wav) will_ogg=false ;; # engine "feature"
 294                 *_bump) will_dds=false ;;
 295                 ./models/player/*) will_dds=false ;;
 296                 ./models/sprites/*) will_dds=false ;;
 297                 ./textures/*) ;;
 298                 ./models/*) ;;
 299                 ./particles/*) ;;
 300                 ./progs/*) ;;
 301                 *)
 302                         # we can't DDS compress the 2D textures, sorry
 303                         # but JPEG is still fine
 304                         will_dds=false
 305                         ;;
 306         esac
 307
 308         # for deluxemaps, lightmaps and normalmaps, enforce high jpeg quality (like on alpha channels)
 309         if [ "$jqual_a" -gt "$jqual_rgb" ]; then
 310                 case "$f" in
 311                         ./maps/*/lm_[0-9][0-9][0-9][13579]) # deluxemap
 312                                 jqual_rgb=$jqual_a
 313                                 ;;
 314                         ./maps/*/lm_[0-9][0-9][0-9][02468]) # lightmap
 315                                 jqual_rgb=$jqual_a
 316                                 ;;
 317                         *_norm) # normalmap
 318                                 jqual_rgb=$jqual_a
 319                                 ;;
 320                 esac
 321         fi
 322
 323         pm=
 324         case "$f" in
 325                 ./particles/particlefont) # particlefont uses premultiplied alpha
 326                         pm=_premul
 327                         ;;
 328         esac
 329
 330         if $do_jpeg_if_not_dds; then
 331                 if $will_dds; then
 332                         will_jpeg=false
 333                 else
 334                         will_jpeg=true
 335                 fi
 336         fi
 337         selfprofile startconvert
 338         case "$F" in
 339                 *_alpha.jpg)
 340                         # handle in *.jpg case
 341
 342                         # they always got converted, I assume
 343                         if $will_dds || $will_jpeg; then
 344                                 conv=true
 345                         fi
 346                         keep=$will_jpeg
 347                         ;;
 348                 *.jpg)
 349                         if [ -f "${f}_alpha.jpg" ]; then
 350                                 cached "$will_dds"  reduce_jpeg2_dds$pm "$F" "${f}_alpha.jpg" "dds/${f}.dds" ""               "$dds_flags"
 351                                 cached "$will_jpeg" reduce_jpeg2_jpeg2  "$F" "${f}_alpha.jpg" "$F"           "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
 352                         else
 353                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 354                                 cached "$will_jpeg" reduce_jpeg_jpeg    "$F" ""               "$F"           ""               "$jqual_rgb"
 355                         fi
 356                         ;;
 357                 *.png|*.tga)
 358                         cached true has_alpha "$F" "" "$F.hasalpha" ""
 359                         conv=false
 360                         if [ -s "$F.hasalpha" ]; then
 361                                 cached "$will_dds"  reduce_rgba_dds$pm  "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 362                                 cached "$will_jpeg" reduce_rgba_jpeg2   "$F" ""               "${f}.jpg"     "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
 363                         else
 364                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 365                                 cached "$will_jpeg" reduce_rgb_jpeg     "$F" ""               "${f}.jpg"     ""               "$jqual_rgb"
 366                         fi
 367                         rm -f "$F.hasalpha"
 368                         ;;
 369                 *.ogg)
 370                         cached "$will_ogg" reduce_ogg_ogg "$F" "" "$F" "" "$ogg_qual"
 371                         ;;
 372                 ./sound/misc/null.wav)
 373                         # never convert this one
 374                         ;;
 375                 *.wav)
 376                         cached "$will_ogg" reduce_wav_ogg "$F" "" "${f}.ogg" "" "$ogg_qual"
 377                         ;;
 378         esac
 379         selfprofile marktodelete
 380         if $del_src; then
 381                 if $conv; then
 382                         if ! $keep; then
 383                                 # FIXME can't have spaces in filenames that way
 384                                 to_delete="$to_delete $F"
 385                         fi
 386                 fi
 387         fi
 388         selfprofile symlinkfixing
 389         # fix up DDS paths by a symbolic link
 390         if [ -f "dds/${f}.dds" ]; then
 391                 if [ -z "${f##./textures/*}" ]; then
 392                         if [ -n "${f##./textures/*/*}" ]; then
 393                                 ln -snf "textures/${f#./textures/}.dds" "dds/${f#./textures/}.dds"
 394                         fi
 395                 fi
 396         fi
 397         selfprofile looping
 398 done
 399
 400 for F in $to_delete; do
 401         rm -f "$F"
 402 done
 403 selfprofile finished_time
 404 set | grep ^selfprofile_counter_ >&2