misc/tools/cached-converter.sh

   1 #!/bin/sh
   2
   3 set -e
   4
   5 : ${CACHEDIR:=$HOME/.xonotic-cached-converter}
   6 : ${do_jpeg:=true}
   7 : ${do_jpeg_if_not_dds:=false}
   8 : ${jpeg_qual_rgb:=95}
   9 : ${jpeg_qual_a:=99}
  10 : ${do_dds:=true}
  11 : ${dds_tool:=compressonator-dxtc}
  12 : ${do_ogg:=false}
  13 : ${ogg_qual:=1}
  14 : ${del_src:=false}
  15 : ${git_src_repo:=}
  16 : ${dds_noalpha:=dxt1}
  17 : ${dds_prealpha:=dxt4}
  18 : ${dds_sepalpha:=dxt5}
  19
  20 selfprofile_t0=`date +%s`
  21 selfprofile_step=init
  22 selfprofile()
  23 {
  24         selfprofile_t=`date +%s`
  25         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step+$selfprofile_t))"
  26         selfprofile_step=$1
  27         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step-$selfprofile_t))"
  28         selfprofile_t0=$selfprofile_t
  29 }
  30
  31 me=$0
  32 case "$me" in
  33         */*)
  34                 meprefix=${me%/*}/
  35                 ;;
  36         *)
  37                 meprefix=
  38                 ;;
  39 esac
  40
  41 tmpdir=`mktemp -d -t cached-converter.XXXXXX`
  42 trap 'exit 1' INT
  43 trap 'rm -rf "$tmpdir"' EXIT
  44
  45
  46 use_magnet_to_acquire_checksum_faster()
  47 #             ___________________
  48 #        ,--'' ~~~~~~~^^^~._     '.
  49 #    ,.-' ~~~~~~~~~~^^^^^~~~._._   \
  50 #    |   /^^^^^|    /^^^^^^^^\\ \   \
  51 #  ,/___  <  o>      <  (OO) > _     \
  52 # /'/,         |-         .       ----.\
  53 # |(|-'^^;,-  ,|     __    ^~~^^^^^^^; |\
  54 # \\`  |    <;_    __ |`---  ..-^^/- | ||
  55 #  \`-|Oq-.____`________~='^^|__,/  ' //
  56 #   \ || | |   |  |    \ ..-;|  /    '/
  57 #   | ||#|#|the|==|game!|'^` |/'    /'
  58 #   | \\\\^\***|***|    \ ,,;'     /
  59 #   |  `-=\_\__\___\__..-' ,.- - ,/
  60 #   | . `-_  ------   _,-'^-'^,-'
  61 #   | `-._________..--''^,-''^
  62 #   \             ,...-'^
  63 #    `----------'^              PROBLEM?
  64 {
  65         magnet=`GIT_DIR="$git_src_repo/.git" git ls-files -s "$1"`
  66         if [ -n "$magnet" ]; then
  67                 magnet=${magnet#* }
  68                 magnet=${magnet%% *}
  69                 sum=$sum$magnet
  70         else
  71                 sum=$sum`git hash-object "$1"`
  72         fi
  73 }
  74
  75 lastinfiles=
  76 lastinfileshash=
  77 cached()
  78 {
  79         flag=$1; shift
  80         method=$1; shift
  81         infile1=$1; shift
  82         infile2=$1; shift
  83         outfile1=$1; shift
  84         outfile2=$1; shift
  85         if ! $flag; then
  86                 return 0
  87         fi
  88         #sleep 0.25
  89         if [ x"$infile1" = x"$outfile1" ]; then
  90                 keep=true
  91         fi
  92         options=`echo "$*" | git hash-object --stdin`
  93         selfprofile convert_findchecksum
  94         if [ x"$infile1/../$infile2" = x"$lastinfiles" ]; then
  95                 sum=$lastinfileshash
  96         else
  97                 evil=false
  98                 for infile in "$infile1" "$infile2"; do
  99                         case "$infile" in
 100                                 */background_l2.tga|*/background_ingame_l2.tga)
 101                                         evil=true
 102                                         ;;
 103                         esac
 104                 done
 105                 if [ -n "$git_src_repo" ] && ! $evil; then
 106                         sum=
 107                         use_magnet_to_acquire_checksum_faster "${infile1#./}"
 108                         if [ -n "$infile2" ]; then
 109                                 use_magnet_to_acquire_checksum_faster "${infile2#./}"
 110                         fi
 111                 else
 112                         sum=`git hash-object "$infile1"`
 113                         if [ -n "$infile2" ]; then
 114                                 sum=$sum`git hash-object "$infile2"`
 115                         fi
 116                 fi
 117                 lastinfileshash=$sum
 118         fi
 119         selfprofile convert_makecachedir
 120         mkdir -p "$CACHEDIR/$method-$options"
 121         name1="$CACHEDIR/$method-$options/$sum-1.${outfile1##*.}"
 122         [ -z "$outfile2" ] || name2="$CACHEDIR/$method-$options/$sum-2.${outfile2##*.}"
 123         tempfile1="${name1%/*}/new-${name1##*/}"
 124         [ -z "$outfile2" ] || tempfile2="${name2%/*}/new-${name2##*/}"
 125         if [ -f "$name1" ] && { [ -z "$outfile2" ] || [ -f "$name2" ]; }; then
 126                 selfprofile convert_copyoutput
 127                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
 128                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
 129                 conv=true
 130         elif selfprofile convert_makeoutput; "$method" "$infile1" "$infile2" "$tempfile1" "$tempfile2" "$@"; then
 131                 mv "$tempfile1" "$name1"
 132                 [ -z "$outfile2" ] || mv "$tempfile2" "$name2"
 133                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
 134                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
 135                 conv=true
 136         else
 137                 selfprofile convert_cleartemp
 138                 rm -f "$tempfile1"
 139                 rm -f "$tempfile2"
 140                 selfprofile convert_finished
 141                 exit 1
 142         fi
 143         selfprofile convert_finished
 144 }
 145
 146 reduce_jpeg2_dds()
 147 {
 148         i=$1; shift
 149         ia=$1; shift
 150         o=$1; shift; shift
 151         convert "$i" "$ia" -compose CopyOpacity -composite "$tmpdir/x.tga" && \
 152         "$meprefix"compress-texture "$dds_tool" "$dds_sepalpha" "$tmpdir/x.tga" "$o" $1
 153 }
 154
 155 reduce_jpeg2_dds_premul()
 156 {
 157         i=$1; shift
 158         ia=$1; shift
 159         o=$1; shift; shift
 160         convert "$i" "$ia" -compose CopyOpacity -composite "$tmpdir/x.tga" && \
 161         "$meprefix"compress-texture "$dds_tool" "$dds_prealpha" "$tmpdir/x.tga" "$o" $1
 162 }
 163
 164 reduce_jpeg2_jpeg2()
 165 {
 166         i=$1; shift
 167         ia=$1; shift
 168         o=$1; shift
 169         oa=$1; shift
 170         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 171                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
 172                         cp "$i" "$o"
 173                 fi
 174         else
 175                 return 1
 176         fi
 177         if convert "$ia" TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
 178                 if [ "`stat -c %s "$ia"`" -lt "`stat -c %s "$oa"`" ]; then
 179                         cp "$ia" "$oa"
 180                 fi
 181         else
 182                 return 1
 183         fi
 184 }
 185
 186 reduce_jpeg_jpeg()
 187 {
 188         i=$1; shift; shift
 189         o=$1; shift; shift
 190         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 191                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
 192                         cp "$i" "$o"
 193                 fi
 194         else
 195                 return 1
 196         fi
 197 }
 198
 199 reduce_ogg_ogg()
 200 {
 201         i=$1; shift; shift
 202         o=$1; shift; shift
 203         tags=`vorbiscomment -R -l "$i" || true`
 204         oggdec -o "$tmpdir/x.wav" "$i" && \
 205         oggenc -q"$1" -o "$o" "$tmpdir/x.wav"
 206         echo "$tags" | vorbiscomment -R -w "$o" || true
 207 }
 208
 209 reduce_wav_ogg()
 210 {
 211         i=$1; shift; shift
 212         o=$1; shift; shift
 213         oggenc -q"$1" -o "$o" "$i"
 214 }
 215
 216 reduce_rgba_dds()
 217 {
 218         i=$1; shift; shift
 219         o=$1; shift; shift
 220         convert "$i" "$tmpdir/x.tga" && \
 221         "$meprefix"compress-texture "$dds_tool" "$dds_sepalpha" "$tmpdir/x.tga" "$o" $1
 222 }
 223
 224 reduce_rgba_dds_premul()
 225 {
 226         i=$1; shift; shift
 227         o=$1; shift; shift
 228         convert "$i" "$tmpdir/x.tga" && \
 229         "$meprefix"compress-texture "$dds_tool" "$dds_prealpha" "$tmpdir/x.tga" "$o" $1
 230 }
 231
 232 reduce_rgba_jpeg2()
 233 {
 234         i=$1; shift; shift
 235         o=$1; shift
 236         oa=$1; shift
 237         if convert "$i" -alpha off TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 238                 :
 239         else
 240                 return 1
 241         fi
 242         if convert "$i" -alpha extract TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
 243                 :
 244         else
 245                 return 1
 246         fi
 247 }
 248
 249 reduce_rgb_dds()
 250 {
 251         i=$1; shift; shift
 252         o=$1; shift; shift
 253         convert "$i" "$tmpdir/x.tga" && \
 254         "$meprefix"compress-texture "$dds_tool" "$dds_noalpha" "$tmpdir/x.tga" "$o" $1
 255 }
 256
 257 reduce_rgb_jpeg()
 258 {
 259         i=$1; shift; shift
 260         o=$1; shift; shift
 261         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 262                 :
 263         else
 264                 return 1
 265         fi
 266 }
 267
 268 has_alpha()
 269 {
 270         i=$1; shift; shift
 271         o=$1; shift; shift
 272         if convert "$i" -depth 16 RGBA:- | perl -e 'while(read STDIN, $_, 8) { substr($_, 6, 2) eq "\xFF\xFF" or exit 1; } exit 0;'; then
 273                 # no alpha
 274                 : > "$o"
 275         else
 276                 # has alpha
 277                 echo yes > "$o"
 278         fi
 279 }
 280
 281 to_delete=
 282 for F in "$@"; do
 283         selfprofile prepareconvert
 284         f=${F%.*}
 285
 286         echo >&2 "Handling $F..."
 287         conv=false
 288         keep=false
 289         jqual_rgb=$jpeg_qual_rgb
 290         jqual_a=$jpeg_qual_a
 291
 292         will_jpeg=$do_jpeg
 293         will_dds=$do_dds
 294         will_ogg=$do_ogg
 295         case "$f" in
 296                 ./sounds/misc/talk*.wav) will_ogg=false ;; # engine "feature"
 297                 *_bump) will_dds=false ;;
 298                 *_norm) will_dds=false ;; # bad quality
 299                 ./models/player/*) will_dds=false ;;
 300                 ./models/sprites/*) will_dds=false ;;
 301                 ./textures/*) ;;
 302                 ./models/*) ;;
 303                 ./particles/*) ;;
 304                 ./progs/*) ;;
 305                 *)
 306                         # we can't DDS compress the 2D textures, sorry
 307                         # but JPEG is still fine
 308                         will_dds=false
 309                         ;;
 310         esac
 311
 312         # for deluxemaps, lightmaps and normalmaps, enforce high jpeg quality (like on alpha channels)
 313         if [ "$jqual_a" -gt "$jqual_rgb" ]; then
 314                 case "$f" in
 315                         ./maps/*/lm_[0-9][0-9][0-9][13579]) # deluxemap
 316                                 jqual_rgb=$jqual_a
 317                                 ;;
 318                         ./maps/*/lm_[0-9][0-9][0-9][02468]) # lightmap
 319                                 jqual_rgb=$jqual_a
 320                                 ;;
 321                         *_norm) # normalmap
 322                                 jqual_rgb=$jqual_a
 323                                 ;;
 324                 esac
 325         fi
 326
 327         pm=
 328         case "$f" in
 329                 ./particles/particlefont) # particlefont uses premultiplied alpha
 330                         pm=_premul
 331                         ;;
 332         esac
 333
 334         if $do_jpeg_if_not_dds; then
 335                 if $will_dds; then
 336                         will_jpeg=false
 337                 else
 338                         will_jpeg=true
 339                 fi
 340         fi
 341         selfprofile startconvert
 342         case "$F" in
 343                 *_alpha.jpg)
 344                         # handle in *.jpg case
 345
 346                         # they always got converted, I assume
 347                         if $will_dds || $will_jpeg; then
 348                                 conv=true
 349                         fi
 350                         keep=$will_jpeg
 351                         ;;
 352                 *.jpg)
 353                         if [ -f "${f}_alpha.jpg" ]; then
 354                                 cached "$will_dds"  reduce_jpeg2_dds$pm "$F" "${f}_alpha.jpg" "dds/${f}.dds" ""               "$dds_flags"
 355                                 cached "$will_jpeg" reduce_jpeg2_jpeg2  "$F" "${f}_alpha.jpg" "$F"           "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
 356                         else
 357                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 358                                 cached "$will_jpeg" reduce_jpeg_jpeg    "$F" ""               "$F"           ""               "$jqual_rgb"
 359                         fi
 360                         ;;
 361                 *.png|*.tga)
 362                         cached true has_alpha "$F" "" "$F.hasalpha" ""
 363                         conv=false
 364                         if [ -s "$F.hasalpha" ]; then
 365                                 cached "$will_dds"  reduce_rgba_dds$pm  "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 366                                 cached "$will_jpeg" reduce_rgba_jpeg2   "$F" ""               "${f}.jpg"     "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
 367                         else
 368                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 369                                 cached "$will_jpeg" reduce_rgb_jpeg     "$F" ""               "${f}.jpg"     ""               "$jqual_rgb"
 370                         fi
 371                         rm -f "$F.hasalpha"
 372                         ;;
 373                 *.ogg)
 374                         cached "$will_ogg" reduce_ogg_ogg "$F" "" "$F" "" "$ogg_qual"
 375                         ;;
 376                 ./sound/misc/null.wav)
 377                         # never convert this one
 378                         ;;
 379                 *.wav)
 380                         cached "$will_ogg" reduce_wav_ogg "$F" "" "${f}.ogg" "" "$ogg_qual"
 381                         ;;
 382         esac
 383         selfprofile marktodelete
 384         if $del_src; then
 385                 if $conv; then
 386                         if ! $keep; then
 387                                 # FIXME can't have spaces in filenames that way
 388                                 to_delete="$to_delete $F"
 389                         fi
 390                 fi
 391         fi
 392         selfprofile symlinkfixing
 393         # fix up DDS paths by a symbolic link
 394         if [ -f "dds/${f}.dds" ]; then
 395                 if [ -z "${f##./textures/*}" ]; then
 396                         if [ -n "${f##./textures/*/*}" ]; then
 397                                 ln -snf "textures/${f#./textures/}.dds" "dds/${f#./textures/}.dds"
 398                         fi
 399                 fi
 400         fi
 401         selfprofile looping
 402 done
 403
 404 for F in $to_delete; do
 405         rm -f "$F"
 406 done
 407 selfprofile finished_time
 408 set | grep ^selfprofile_counter_ >&2