misc/tools/cached-converter.sh

   1 #!/bin/sh
   2
   3 set -e
   4
   5 : ${CACHEDIR:=$HOME/.xonotic-cached-converter}
   6 : ${do_jpeg:=true}
   7 : ${do_jpeg_if_not_dds:=false}
   8 : ${jpeg_qual_rgb:=95}
   9 : ${jpeg_qual_a:=99}
  10 : ${do_dds:=true}
  11 : ${dds_tool:=compressonator-dxtc}
  12 : ${do_ogg:=false}
  13 : ${ogg_qual:=1}
  14 : ${del_src:=false}
  15 : ${git_src_repo:=}
  16
  17 selfprofile_t0=`date +%s`
  18 selfprofile_step=init
  19 selfprofile()
  20 {
  21         selfprofile_t=`date +%s`
  22         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step+$selfprofile_t))"
  23         selfprofile_step=$1
  24         eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step-$selfprofile_t))"
  25         selfprofile_t0=$selfprofile_t
  26 }
  27
  28 me=$0
  29 case "$me" in
  30         */*)
  31                 meprefix=${me%/*}/
  32                 ;;
  33         *)
  34                 meprefix=
  35                 ;;
  36 esac
  37
  38 tmpdir=`mktemp -d -t cached-converter.XXXXXX`
  39 trap 'exit 1' INT
  40 trap 'rm -rf "$tmpdir"' EXIT
  41
  42 lastinfiles=
  43 lastinfileshash=
  44 cached()
  45 {
  46         flag=$1; shift
  47         method=$1; shift
  48         infile1=$1; shift
  49         infile2=$1; shift
  50         outfile1=$1; shift
  51         outfile2=$1; shift
  52         if ! $flag; then
  53                 return 0
  54         fi
  55         #sleep 0.25
  56         if [ x"$infile1" = x"$outfile1" ]; then
  57                 keep=true
  58         fi
  59         options=`echo "$*" | git hash-object --stdin`
  60         selfprofile convert_findchecksum
  61         if [ x"$infile1/../$infile2" = x"$lastinfiles" ]; then
  62                 sum=$lastinfileshash
  63         else
  64                 evil=false
  65                 for infile in "$infile1" "$infile2"; do
  66                         case "$infile" in
  67                                 */background_l2.tga|*/background_ingame_l2.tga)
  68                                         evil=true
  69                                         ;;
  70                         esac
  71                 done
  72                 if [ -n "$git_src_repo" ] && ! $evil; then
  73                         sum=`( cd "$git_src_repo"; git ls-files -s "${infile1#./}" | cut -d ' ' -f 2 | grep . ) || { echo >&2 "git-ls-files failed on $infile1"; git hash-object "$infile1"; }`
  74                         if [ -n "$infile2" ]; then
  75                                 sum=$sum`( cd "$git_src_repo"; git ls-files -s "${infile2#./}" | cut -d ' ' -f 2 | grep . ) || { echo >&2 "git-ls-files failed on $infile2"; git hash-object "$infile2"; }`
  76                         fi
  77                 else
  78                         sum=`git hash-object "$infile1"`
  79                         if [ -n "$infile2" ]; then
  80                                 sum=$sum`git hash-object "$infile2"`
  81                         fi
  82                 fi
  83                 lastinfileshash=$sum
  84         fi
  85         selfprofile convert_makecachedir
  86         mkdir -p "$CACHEDIR/$method-$options"
  87         name1="$CACHEDIR/$method-$options/$sum-1.${outfile1##*.}"
  88         [ -z "$outfile2" ] || name2="$CACHEDIR/$method-$options/$sum-2.${outfile2##*.}"
  89         tempfile1="${name1%/*}/new-${name1##*/}"
  90         [ -z "$outfile2" ] || tempfile2="${name2%/*}/new-${name2##*/}"
  91         if [ -f "$name1" ] && { [ -z "$outfile2" ] || [ -f "$name2" ]; }; then
  92                 selfprofile convert_copyoutput
  93                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
  94                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
  95                 conv=true
  96         elif selfprofile convert_makeoutput; "$method" "$infile1" "$infile2" "$tempfile1" "$tempfile2" "$@"; then
  97                 mv "$tempfile1" "$name1"
  98                 [ -z "$outfile2" ] || mv "$tempfile2" "$name2"
  99                 case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
 100                 [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
 101                 conv=true
 102         else
 103                 selfprofile convert_cleartemp
 104                 rm -f "$tempfile1"
 105                 rm -f "$tempfile2"
 106                 selfprofile convert_finished
 107                 exit 1
 108         fi
 109         selfprofile convert_finished
 110 }
 111
 112 reduce_jpeg2_dds()
 113 {
 114         i=$1; shift
 115         ia=$1; shift
 116         o=$1; shift; shift
 117         convert "$i" "$ia" -compose CopyOpacity -composite "$tmpdir/x.tga" && \
 118         "$meprefix"compress-texture "$dds_tool" dxt5 "$tmpdir/x.tga" "$o" $1
 119 }
 120
 121 reduce_jpeg2_dds_premul()
 122 {
 123         i=$1; shift
 124         ia=$1; shift
 125         o=$1; shift; shift
 126         convert "$i" "$ia" -compose CopyOpacity -composite "$tmpdir/x.tga" && \
 127         "$meprefix"compress-texture "$dds_tool" dxt4 "$tmpdir/x.tga" "$o" $1
 128 }
 129
 130 reduce_jpeg2_jpeg2()
 131 {
 132         i=$1; shift
 133         ia=$1; shift
 134         o=$1; shift
 135         oa=$1; shift
 136         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 137                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
 138                         cp "$i" "$o"
 139                 fi
 140         else
 141                 return 1
 142         fi
 143         if convert "$ia" TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
 144                 if [ "`stat -c %s "$ia"`" -lt "`stat -c %s "$oa"`" ]; then
 145                         cp "$ia" "$oa"
 146                 fi
 147         else
 148                 return 1
 149         fi
 150 }
 151
 152 reduce_jpeg_jpeg()
 153 {
 154         i=$1; shift; shift
 155         o=$1; shift; shift
 156         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 157                 if [ "`stat -c %s "$i"`" -lt "`stat -c %s "$o"`" ]; then
 158                         cp "$i" "$o"
 159                 fi
 160         else
 161                 return 1
 162         fi
 163 }
 164
 165 reduce_ogg_ogg()
 166 {
 167         i=$1; shift; shift
 168         o=$1; shift; shift
 169         tags=`vorbiscomment -R -l "$i" || true`
 170         oggdec -o "$tmpdir/x.wav" "$i" && \
 171         oggenc -q"$1" -o "$o" "$tmpdir/x.wav"
 172         echo "$tags" | vorbiscomment -R -w "$o" || true
 173 }
 174
 175 reduce_wav_ogg()
 176 {
 177         i=$1; shift; shift
 178         o=$1; shift; shift
 179         oggenc -q"$1" -o "$o" "$i"
 180 }
 181
 182 reduce_rgba_dds()
 183 {
 184         i=$1; shift; shift
 185         o=$1; shift; shift
 186         convert "$i" "$tmpdir/x.tga" && \
 187         "$meprefix"compress-texture "$dds_tool" dxt5 "$tmpdir/x.tga" "$o" $1
 188 }
 189
 190 reduce_rgba_dds_premul()
 191 {
 192         i=$1; shift; shift
 193         o=$1; shift; shift
 194         convert "$i" "$tmpdir/x.tga" && \
 195         "$meprefix"compress-texture "$dds_tool" dxt4 "$tmpdir/x.tga" "$o" $1
 196 }
 197
 198 reduce_rgba_jpeg2()
 199 {
 200         i=$1; shift; shift
 201         o=$1; shift
 202         oa=$1; shift
 203         if convert "$i" -alpha off TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 204                 :
 205         else
 206                 return 1
 207         fi
 208         if convert "$i" -alpha extract TGA:- | cjpeg -targa -quality "$2" -optimize -sample 1x1,1x1,1x1 > "$oa"; then
 209                 :
 210         else
 211                 return 1
 212         fi
 213 }
 214
 215 reduce_rgb_dds()
 216 {
 217         i=$1; shift; shift
 218         o=$1; shift; shift
 219         convert "$i" "$tmpdir/x.tga" && \
 220         "$meprefix"compress-texture "$dds_tool" dxt1 "$tmpdir/x.tga" "$o" $1
 221 }
 222
 223 reduce_rgb_jpeg()
 224 {
 225         i=$1; shift; shift
 226         o=$1; shift; shift
 227         if convert "$i" TGA:- | cjpeg -targa -quality "$1" -optimize -sample 1x1,1x1,1x1 > "$o"; then
 228                 :
 229         else
 230                 return 1
 231         fi
 232 }
 233
 234 has_alpha()
 235 {
 236         i=$1; shift; shift
 237         o=$1; shift; shift
 238         if convert "$i" -depth 16 RGBA:- | perl -e 'while(read STDIN, $_, 8) { substr($_, 6, 2) eq "\xFF\xFF" or exit 1; } exit 0;'; then
 239                 # no alpha
 240                 : > "$o"
 241         else
 242                 # has alpha
 243                 echo yes > "$o"
 244         fi
 245 }
 246
 247 to_delete=
 248 for F in "$@"; do
 249         selfprofile prepareconvert
 250         f=${F%.*}
 251
 252         echo >&2 "Handling $F..."
 253         conv=false
 254         keep=false
 255         jqual_rgb=$jpeg_qual_rgb
 256         jqual_a=$jpeg_qual_a
 257
 258         will_jpeg=$do_jpeg
 259         will_dds=$do_dds
 260         case "$f" in
 261                 *_bump) will_dds=false ;;
 262                 ./models/player/*) will_dds=false ;;
 263                 ./models/sprites/*) will_dds=false ;;
 264                 ./textures/*) ;;
 265                 ./models/*) ;;
 266                 ./particles/*) ;;
 267                 ./progs/*) ;;
 268                 *)
 269                         # we can't DDS compress the 2D textures, sorry
 270                         # but JPEG is still fine
 271                         will_dds=false
 272                         ;;
 273         esac
 274
 275         # for deluxemaps, lightmaps and normalmaps, enforce high jpeg quality (like on alpha channels)
 276         if [ "$jqual_a" -gt "$jqual_rgb" ]; then
 277                 case "$f" in
 278                         ./maps/*/lm_[0-9][0-9][0-9][13579]) # deluxemap
 279                                 jqual_rgb=$jqual_a
 280                                 ;;
 281                         ./maps/*/lm_[0-9][0-9][0-9][02468]) # lightmap
 282                                 jqual_rgb=$jqual_a
 283                                 ;;
 284                         *_norm) # normalmap
 285                                 jqual_rgb=$jqual_a
 286                                 ;;
 287                 esac
 288         fi
 289
 290         pm=
 291         case "$f" in
 292                 ./particles/particlefont) # particlefont uses premultiplied alpha
 293                         pm=_premul
 294                         ;;
 295         esac
 296
 297         if $do_jpeg_if_not_dds; then
 298                 if $will_dds; then
 299                         will_jpeg=false
 300                 else
 301                         will_jpeg=true
 302                 fi
 303         fi
 304         selfprofile startconvert
 305         case "$F" in
 306                 *_alpha.jpg)
 307                         # handle in *.jpg case
 308
 309                         # they always got converted, I assume
 310                         if $will_dds || $will_jpeg; then
 311                                 conv=true
 312                         fi
 313                         keep=$will_jpeg
 314                         ;;
 315                 *.jpg)
 316                         if [ -f "${f}_alpha.jpg" ]; then
 317                                 cached "$will_dds"  reduce_jpeg2_dds$pm "$F" "${f}_alpha.jpg" "dds/${f}.dds" ""               "$dds_flags"
 318                                 cached "$will_jpeg" reduce_jpeg2_jpeg2  "$F" "${f}_alpha.jpg" "$F"           "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
 319                         else
 320                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 321                                 cached "$will_jpeg" reduce_jpeg_jpeg    "$F" ""               "$F"           ""               "$jqual_rgb"
 322                         fi
 323                         ;;
 324                 *.png|*.tga)
 325                         cached true has_alpha "$F" "" "$F.hasalpha" ""
 326                         conv=false
 327                         if [ -s "$F.hasalpha" ]; then
 328                                 cached "$will_dds"  reduce_rgba_dds$pm  "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 329                                 cached "$will_jpeg" reduce_rgba_jpeg2   "$F" ""               "${f}.jpg"     "${f}_alpha.jpg" "$jqual_rgb" "$jqual_a"
 330                         else
 331                                 cached "$will_dds"  reduce_rgb_dds      "$F" ""               "dds/${f}.dds" ""               "$dds_flags"
 332                                 cached "$will_jpeg" reduce_rgb_jpeg     "$F" ""               "${f}.jpg"     ""               "$jqual_rgb"
 333                         fi
 334                         rm -f "$F.hasalpha"
 335                         ;;
 336                 *.ogg)
 337                         cached "$do_ogg" reduce_ogg_ogg "$F" "" "$F" "" "$ogg_qual"
 338                         ;;
 339                 ./sound/misc/null.wav)
 340                         # never convert this one
 341                         ;;
 342                 *.wav)
 343                         cached "$do_ogg" reduce_wav_ogg "$F" "" "${f}.ogg" "" "$ogg_qual"
 344                         ;;
 345         esac
 346         selfprofile marktodelete
 347         if $del_src; then
 348                 if $conv; then
 349                         if ! $keep; then
 350                                 # FIXME can't have spaces in filenames that way
 351                                 to_delete="$to_delete $F"
 352                         fi
 353                 fi
 354         fi
 355         selfprofile symlinkfixing
 356         # fix up DDS paths by a symbolic link
 357         if [ -f "dds/${f}.dds" ]; then
 358                 if [ -z "${f##./textures/*}" ]; then
 359                         if [ -n "${f##./textures/*/*}" ]; then
 360                                 ln -snf "textures/${f#./textures/}.dds" "dds/${f#./textures/}.dds"
 361                         fi
 362                 fi
 363         fi
 364         selfprofile looping
 365 done
 366
 367 for F in $to_delete; do
 368         rm -f "$F"
 369 done
 370 selfprofile finished_time
 371 set | grep ^selfprofile_counter_ >&2