]> git.xonotic.org Git - xonotic/xonotic.git/blobdiff - misc/tools/cached-converter.sh
fix more bugs
[xonotic/xonotic.git] / misc / tools / cached-converter.sh
index 517ad9822755f61a7e98513dfa9d74d4aa4bb448..aa4089c1273e42f1db5efa84ac899883a72bf2b1 100755 (executable)
@@ -13,6 +13,20 @@ set -e
 : ${ogg_qual:=1}
 : ${del_src:=false}
 : ${git_src_repo:=}
+: ${dds_noalpha:=dxt1}
+: ${dds_prealpha:=dxt2 dxt4}
+: ${dds_sepalpha:=dxt3 dxt5}
+
+selfprofile_t0=`date +%s`
+selfprofile_step=init
+selfprofile()
+{
+       selfprofile_t=`date +%s`
+       eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step+$selfprofile_t))"
+       selfprofile_step=$1
+       eval "selfprofile_counter_$selfprofile_step=\$((\$selfprofile_counter_$selfprofile_step-$selfprofile_t))"
+       selfprofile_t0=$selfprofile_t
+}
 
 me=$0
 case "$me" in
@@ -28,8 +42,68 @@ tmpdir=`mktemp -d -t cached-converter.XXXXXX`
 trap 'exit 1' INT
 trap 'rm -rf "$tmpdir"' EXIT
 
+
+use_magnet_to_acquire_checksum_faster()
+#             ___________________
+#        ,--'' ~~~~~~~^^^~._     '.
+#    ,.-' ~~~~~~~~~~^^^^^~~~._._   \
+#    |   /^^^^^|    /^^^^^^^^\\ \   \
+#  ,/___  <  o>      <  (OO) > _     \
+# /'/,         |-         .       ----.\
+# |(|-'^^;,-  ,|     __    ^~~^^^^^^^; |\
+# \\`  |    <;_    __ |`---  ..-^^/- | ||
+#  \`-|Oq-.____`________~='^^|__,/  ' //
+#   \ || | |   |  |    \ ..-;|  /    '/
+#   | ||#|#|the|==|game!|'^` |/'    /'
+#   | \\\\^\***|***|    \ ,,;'     /
+#   |  `-=\_\__\___\__..-' ,.- - ,/
+#   | . `-_  ------   _,-'^-'^,-'
+#   | `-._________..--''^,-''^
+#   \             ,...-'^
+#    `----------'^              PROBLEM?
+{
+       magnet=`GIT_DIR="$git_src_repo/.git" git ls-files -s "$1"`
+       if [ -n "$magnet" ]; then
+               magnet=${magnet#* }
+               magnet=${magnet%% *}
+               echo "$magnet"
+       else
+               git hash-object "$1"
+       fi
+}
+
 lastinfiles=
 lastinfileshash=
+acquire_checksum()
+{
+       if [ x"$1/../$2" = x"$lastinfiles" ]; then
+               _a_s=$lastinfileshash
+       else
+               _a_e=false
+               for _a_f in "$1" "$2"; do
+                       case "$_a_f" in
+                               */background_l2.tga|*/background_ingame_l2.tga)
+                                       _a_e=true
+                                       ;;
+                       esac
+               done
+               if [ -n "$git_src_repo" ] && ! $_a_e; then
+                       _a_s=`use_magnet_to_acquire_checksum_faster "${1#./}"`
+                       if [ -n "$2" ]; then
+                               _a_s=$_a_s`use_magnet_to_acquire_checksum_faster "${2#./}"`
+                       fi
+               else
+                       _a_s=`git hash-object "$1"`
+                       if [ -n "$2" ]; then
+                               _a_s=$_a_s`git hash-object "$2"`
+                       fi
+               fi
+               lastinfileshash=$_a_s
+               lastinfiles="$1/../$2"
+       fi
+       echo "$_a_s"
+}
+
 cached()
 {
        flag=$1; shift
@@ -46,42 +120,82 @@ cached()
                keep=true
        fi
        options=`echo "$*" | git hash-object --stdin`
-       if [ x"$infile1/../$infile2" = x"$lastinfiles" ]; then
-               sum=$lastinfileshash
-       else
-               if [ -n "$git_src_repo" ]; then
-                       sum=`( cd "$git_src_repo"; git rev-parse --revs-only HEAD:"${infile1#./}" | grep . ) || git hash-object "$infile1"`
-                       if [ -n "$infile2" ]; then
-                               sum=$sum`( cd "$git_src_repo"; git rev-parse --revs-only HEAD:"${infile2#./}" | grep . ) || git hash-object "$infile2"`
-                       fi
-               else
-                       sum=`git hash-object "$infile1"`
-                       if [ -n "$infile2" ]; then
-                               sum=$sum`git hash-object "$infile2"`
-                       fi
-               fi
-               lastinfileshash=$sum
-       fi
+       selfprofile convert_findchecksum
+       sum=`acquire_checksum "$infile1" "$infile2"`
+       selfprofile convert_makecachedir
        mkdir -p "$CACHEDIR/$method-$options"
        name1="$CACHEDIR/$method-$options/$sum-1.${outfile1##*.}"
        [ -z "$outfile2" ] || name2="$CACHEDIR/$method-$options/$sum-2.${outfile2##*.}"
        tempfile1="${name1%/*}/new-${name1##*/}"
        [ -z "$outfile2" ] || tempfile2="${name2%/*}/new-${name2##*/}"
        if [ -f "$name1" ] && { [ -z "$outfile2" ] || [ -f "$name2" ]; }; then
-               case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln "$name1" "$outfile1" 2>/dev/null || cp "$name1" "$outfile1"; }
-               [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln "$name2" "$outfile2" 2>/dev/null || cp "$name2" "$outfile2"; }; }
+               selfprofile convert_copyoutput
+               case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
+               [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
                conv=true
-       elif "$method" "$infile1" "$infile2" "$tempfile1" "$tempfile2" "$@"; then
+       elif selfprofile convert_makeoutput; "$method" "$infile1" "$infile2" "$tempfile1" "$tempfile2" "$@"; then
                mv "$tempfile1" "$name1"
                [ -z "$outfile2" ] || mv "$tempfile2" "$name2"
-               case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln "$name1" "$outfile1" 2>/dev/null || cp "$name1" "$outfile1"; }
-               [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln "$name2" "$outfile2" 2>/dev/null || cp "$name2" "$outfile2"; }; }
+               case "$outfile1" in */*) mkdir -p "${outfile1%/*}"; esac && { ln -f "$name1" "$outfile1" 2>/dev/null || { rm -f "$outfile1" && cp "$name1" "$outfile1"; }; }
+               [ -z "$outfile2" ] || { case "$outfile2" in */*) mkdir -p "${outfile2%/*}"; esac && { ln -f "$name2" "$outfile2" 2>/dev/null || { rm -f "$outfile2" && cp "$name2" "$outfile2"; }; }; }
                conv=true
        else
+               selfprofile convert_cleartemp
                rm -f "$tempfile1"
                rm -f "$tempfile2"
+               selfprofile convert_finished
                exit 1
        fi
+       selfprofile convert_finished
+}
+
+pickdxta()
+{
+       pd_t=$1; shift
+       pd_d=$1; shift
+       pd_i=$1; shift
+       pd_o=$1; shift
+       for pd_dd in $pd_d; do
+               if [ -f "$pd_o" ]; then
+                       "$meprefix"compress-texture "$pd_t" "$pd_dd" "$pd_i" "$pd_o".tmp.dds "$@"
+                       pd_psnr_tmp=`compare -channel alpha -metric PSNR "$pd_i" "$pd_o".tmp.dds NULL: 2>&1`
+                       case "$pd_psnr_tmp" in
+                               [0-9.]*)
+                                       ;;
+                               *)
+                                       pd_psnr_tmp=999.9
+                                       ;;
+                       esac
+                       echo >&2 "$pd_dd: $pd_psnr_tmp dB"
+                       pd_psnr_diff=`echo "($pd_psnr_tmp) - ($pd_psnr)" | bc -l`
+                       case "$pd_psnr_diff" in
+                               -*|0)
+                                       # tmp is smaller or equal
+                                       # smaller PSNR is worse
+                                       # no action
+                                       ;;
+                               *)
+                                       # tmp is larger
+                                       # larger PSNR is better
+                                       pd_psnr=$pd_psnr_tmp
+                                       mv "$pd_o".tmp.dds "$pd_o"
+                                       echo >&2 "PICKED (better)"
+                                       ;;
+                       esac
+               else
+                       "$meprefix"compress-texture "$pd_t" "$pd_dd" "$pd_i" "$pd_o" "$@"
+                       pd_psnr=`compare -channel alpha -metric PSNR "$pd_i" "$pd_o" NULL: 2>&1`
+                       case "$pd_psnr" in
+                               [0-9.]*)
+                                       ;;
+                               *)
+                                       pd_psnr=999.9
+                                       ;;
+                       esac
+                       echo >&2 "$pd_dd: $pd_psnr dB"
+                       echo >&2 "PICKED (first)"
+               fi
+       done
 }
 
 reduce_jpeg2_dds()
@@ -90,7 +204,7 @@ reduce_jpeg2_dds()
        ia=$1; shift
        o=$1; shift; shift 
        convert "$i" "$ia" -compose CopyOpacity -composite "$tmpdir/x.tga" && \
-       "$meprefix"compress-texture "$dds_tool" dxt5 "$tmpdir/x.tga" "$o" $1
+       pickdxta "$dds_tool" "$dds_sepalpha" "$tmpdir/x.tga" "$o" $1
 }
 
 reduce_jpeg2_dds_premul()
@@ -99,7 +213,7 @@ reduce_jpeg2_dds_premul()
        ia=$1; shift
        o=$1; shift; shift 
        convert "$i" "$ia" -compose CopyOpacity -composite "$tmpdir/x.tga" && \
-       "$meprefix"compress-texture "$dds_tool" dxt4 "$tmpdir/x.tga" "$o" $1
+       pickdxta "$dds_tool" "$dds_prealpha" "$tmpdir/x.tga" "$o" $1
 }
 
 reduce_jpeg2_jpeg2()
@@ -159,7 +273,7 @@ reduce_rgba_dds()
        i=$1; shift; shift
        o=$1; shift; shift
        convert "$i" "$tmpdir/x.tga" && \
-       "$meprefix"compress-texture "$dds_tool" dxt5 "$tmpdir/x.tga" "$o" $1
+       pickdxta "$dds_tool" "$dds_sepalpha" "$tmpdir/x.tga" "$o" $1
 }
 
 reduce_rgba_dds_premul()
@@ -167,7 +281,7 @@ reduce_rgba_dds_premul()
        i=$1; shift; shift
        o=$1; shift; shift
        convert "$i" "$tmpdir/x.tga" && \
-       "$meprefix"compress-texture "$dds_tool" dxt4 "$tmpdir/x.tga" "$o" $1
+       pickdxta "$dds_tool" "$dds_prealpha" "$tmpdir/x.tga" "$o" $1
 }
 
 reduce_rgba_jpeg2()
@@ -192,7 +306,7 @@ reduce_rgb_dds()
        i=$1; shift; shift
        o=$1; shift; shift
        convert "$i" "$tmpdir/x.tga" && \
-       "$meprefix"compress-texture "$dds_tool" dxt1 "$tmpdir/x.tga" "$o" $1
+       "$meprefix"compress-texture "$dds_tool" "$dds_noalpha" "$tmpdir/x.tga" "$o" $1
 }
 
 reduce_rgb_jpeg()
@@ -221,6 +335,7 @@ has_alpha()
 
 to_delete=
 for F in "$@"; do
+       selfprofile prepareconvert
        f=${F%.*}
 
        echo >&2 "Handling $F..."
@@ -231,7 +346,9 @@ for F in "$@"; do
 
        will_jpeg=$do_jpeg
        will_dds=$do_dds
+       will_ogg=$do_ogg
        case "$f" in
+               ./sounds/misc/talk*.wav) will_ogg=false ;; # engine "feature"
                *_bump) will_dds=false ;;
                ./models/player/*) will_dds=false ;;
                ./models/sprites/*) will_dds=false ;;
@@ -246,6 +363,20 @@ for F in "$@"; do
                        ;;
        esac
 
+       # configure S2TC
+       case "$f" in
+               *_norm)
+                       export S2TC_COLORDIST_MODE=NORMALMAP
+                       export S2TC_RANDOM_COLORS=256
+                       export S2TC_REFINE_COLORS=LOOP
+                       ;;
+               *)
+                       export S2TC_COLORDIST_MODE=SRGB_MIXED
+                       export S2TC_RANDOM_COLORS=64
+                       export S2TC_REFINE_COLORS=LOOP
+                       ;;
+       esac
+
        # for deluxemaps, lightmaps and normalmaps, enforce high jpeg quality (like on alpha channels)
        if [ "$jqual_a" -gt "$jqual_rgb" ]; then
                case "$f" in
@@ -275,7 +406,7 @@ for F in "$@"; do
                        will_jpeg=true
                fi
        fi
-
+       selfprofile startconvert
        case "$F" in
                *_alpha.jpg)
                        # handle in *.jpg case
@@ -308,15 +439,16 @@ for F in "$@"; do
                        rm -f "$F.hasalpha"
                        ;;
                *.ogg)
-                       cached "$do_ogg" reduce_ogg_ogg "$F" "" "$F" "" "$ogg_qual"
+                       cached "$will_ogg" reduce_ogg_ogg "$F" "" "$F" "" "$ogg_qual"
                        ;;
                ./sound/misc/null.wav)
                        # never convert this one
                        ;;
                *.wav)
-                       cached "$do_ogg" reduce_wav_ogg "$F" "" "${f}.ogg" "" "$ogg_qual"
+                       cached "$will_ogg" reduce_wav_ogg "$F" "" "${f}.ogg" "" "$ogg_qual"
                        ;;
        esac
+       selfprofile marktodelete
        if $del_src; then
                if $conv; then
                        if ! $keep; then
@@ -325,6 +457,7 @@ for F in "$@"; do
                        fi
                fi
        fi
+       selfprofile symlinkfixing
        # fix up DDS paths by a symbolic link
        if [ -f "dds/${f}.dds" ]; then
                if [ -z "${f##./textures/*}" ]; then
@@ -333,7 +466,11 @@ for F in "$@"; do
                        fi
                fi
        fi
+       selfprofile looping
 done
+
 for F in $to_delete; do
        rm -f "$F"
 done
+selfprofile finished_time
+set | grep ^selfprofile_counter_ >&2