misc/tools/encode-demos.sh

   1 #!/bin/bash
   2 # name: encode-demos.sh
   3 # version: 0.6.1
   4 # author: Tyler "-z-" Mulligan
   5 # license: GPL & MIT
   6 # date: 24-02-2017
   7 # description: headless encoding of demo files to HD video concurrently with Xfvb and parallel
   8 #
   9 # The encoding is done with a full Xonotic client inside Xfvb.
  10 # parallel is acting as a job queue.
  11 # You may want to create a new userdir such as `~/.xonotic-clean` for encoding.
  12 # If you don't want certain details of your player config carrying over.
  13 #
  14 # The following is a good starting point for 1080p videos:
  15 #
  16 # ```
  17 # // autoexec.cfg
  18 # bgmvolume 1
  19 #
  20 # vid_height 1080
  21 # vid_width 1920
  22 # scr_screenshot_gammaboost 1
  23 # cl_capturevideo_width 1920
  24 # cl_capturevideo_height 1080
  25 # cl_capturevideo_fps 60
  26 # cl_capturevideo_ogg 1
  27 # cl_capturevideo_ogg_theora_quality 63
  28 # cl_capturevideo_ogg_theora_bitrate -1
  29 # cl_capturevideo_ogg_theora_keyframe_bitrate_multiplier 2
  30 # cl_capturevideo_ogg_theora_keyframe_maxinterval 500
  31 # cl_capturevideo_ogg_theora_keyframe_mininterval 1
  32 # cl_capturevideo_ogg_theora_keyframe_auto_threshold 80
  33 # cl_capturevideo_ogg_theora_noise_sensitivity 0
  34 # cl_capturevideo_ogg_vorbis_quality 10
  35 #
  36 # // HUD stuff
  37 # defer 5 "menu_watermark \"\""
  38 # set cl_allow_uid2name 0; set cl_allow_uidtracking 0
  39 # con_notify 0; con_notifysize 0; con_notifytime 0; showspeed 0; showfps 0
  40 # ```
  41 #
  42
  43 # Customize
  44 XONDIR=${HOME}/xonotic/xonotic                      # path to ./all
  45 USERDIR=${HOME}/.xonotic-clean                      # path to Xonotic userdir for client that does encoding
  46 XONOTIC_BIN="./all"                                 # binary used to launch Xonotic
  47 JOB_TIMEOUT="1h"                                    # if demo doesn't quit itself or hangs
  48 JOBS=4                                              # number of concurrent jobs
  49 DEFAULT_DEMO_LIST_FILE="demos.txt"                  # for batch
  50 DISPLAY=:1.0                                        # display for Xvfb
  51 DIMENSIONS=1920x1080                                # dimensions of virtual display
  52 COMPRESS=false                                      # whether to compress by default
  53 KILLER_KEYWORD_WATCHER=true                         # watch server logs for keyword, kill worker if true
  54 KILLER_KEYWORD="Server Disconnected"                # keyword
  55 KILLER_KEYWORD_WAIT="10s"                           # time to wait between polling watchers
  56 LIST_JOBS_FOLLOW_WAIT="10s"                         # how often to poll the job list with -f
  57
  58 # Internal Constants
  59 SCRIPT_NAME=$(basename $0 .sh)
  60 VERSION=$(awk 'NR == 3 {print $3; exit}' $0)
  61 FFMPEG=$(which ffmpeg)
  62 QUEUE_FILE_DEMOS="/tmp/${SCRIPT_NAME}.jobqueue"
  63 QUEUE_FILE_COMPRESSING="/tmp/${SCRIPT_NAME}_compress.jobqueue"
  64 LOCK_FILE="/tmp/${SCRIPT_NAME}.lock"
  65 LOCK_FILE_COMPRESSING="/tmp/${SCRIPT_NAME}.lock"
  66 LOG_FILE="${SCRIPT_NAME}.log"
  67 REGEX_DEMOLIST_FILE="^[[:alnum:]]+\.txt$"
  68 REGEX_DURATION="^[0-9]+(d|h|m|s)$"
  69 REGEX_VIDEO_TYPES="^(mp4|webm)$"
  70
  71 # State
  72 export KILLER_KEYWORD_WATCHING=true
  73
  74 # Data Helpers
  75 ###############
  76
  77 _get_compression_command() {
  78     if [[ ${FFMPEG} == "" ]]; then
  79         echo "[ ERROR ] ffmpeg or avconv required"
  80         exit 1
  81     fi
  82     if [[ ! $1 ]]; then
  83         echo "[ ERROR ] Video name required"
  84         exit 1
  85     fi
  86     local video_file=$1
  87     local type="mp4"
  88     if [[ $2 =~ ${REGEX_VIDEO_TYPES} ]]; then
  89         type=$2
  90     fi
  91     # compress
  92     if [[ ${type} == "mp4" ]]; then
  93         local output_video_file=$(echo ${video_file} | sed 's/\.ogv$/\.mp4/')
  94         command="${FFMPEG} -i ${video_file} -y -codec:v libx264 -crf 21 -bf 2 -flags +cgop -pix_fmt yuv420p -codec:a aac -strict -2 -b:a 384k -r:a 48000 -movflags faststart ${output_video_file}"
  95     elif [[ ${type} == "webm" ]]; then
  96         local output_video_file=$(echo ${video_file} | sed 's/\.ogv$/\.webm/')
  97         command="${FFMPEG} -i ${video_file} -y -acodec libvorbis -aq 5 -ac 2 -qmax 25 -threads 2 ${output_video_file}"
  98     fi
  99     echo ${command}
 100 }
 101
 102 _get_demo_command() {
 103     local demo_file=$1
 104     local index=$2
 105     name_format=$(basename "${demo_file}" .dem)
 106     command="${XONDIR}/${XONOTIC_BIN} run sdl -simsound -sessionid xonotic_${SCRIPT_NAME}_${index} -userdir \"${USERDIR}\" \
 107         +log_file \"xonotic_${SCRIPT_NAME}_${index}_${name_format}.log\" \
 108         +cl_capturevideo_nameformat \"${name_format}_\" \
 109         +cl_capturevideo_number 0 \
 110         +playdemo \"${demo_file}\" \
 111         +toggle cl_capturevideo \
 112         +alias cl_hook_shutdown quit \
 113         > /dev/null 2>&1"
 114     echo ${command}
 115 }
 116
 117 _get_demos_from_file() {
 118     local file=$1
 119     if [[ -f ${file} ]]; then
 120         local lines
 121         OLD_IFS=${IFS}
 122         IFS=$'\n' read -d '' -r -a lines < ${file}
 123         IFS=${OLD_IFS}
 124         echo ${lines[@]}
 125     fi
 126 }
 127
 128 # Queue Helpers
 129 ################
 130
 131 _queue_add_job() {
 132     local queue_file=$1;
 133     local command=$2;
 134     local nice_name=${command};
 135     local nice_queue_name=${queue_file##*/};
 136     if [[ $3 ]]; then
 137         nice_name=$3
 138     fi
 139     echo "[ INFO ] '${nice_queue_name/.jobqueue/}' new job: ${nice_name}"
 140     echo "${command}" >> ${queue_file}
 141 }
 142
 143 _queue_add_compression_jobs() {
 144     local queue_file=$1; shift
 145     local type=$1; shift
 146     local videos="$@"
 147     for video_file in ${videos[@]}; do
 148         local command=$(_get_compression_command ${USERDIR}/data/${video_file} ${type})
 149         _queue_add_job ${queue_file} "${command}" ${video_file}
 150     done
 151 }
 152
 153 _queue_add_demo_jobs() {
 154     local queue_file=$1; shift
 155     local timeout=$1; shift
 156     local demos=$@
 157     local i=0
 158     for demo_file in ${demos[@]}; do
 159         local command=$(_get_demo_command ${demo_file} ${i})
 160         command="timeout ${timeout} ${command}"
 161         _queue_add_job ${queue_file} "${command}" ${demo_file}
 162         ((i++))
 163     done
 164 }
 165
 166 _get_active_demo_jobs() {
 167     if [[ $(pgrep -caf "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_") -gt 0 ]]; then
 168         pgrep -af "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_" |grep "dev/null" |awk '{ print $17 }' |sed 's/"//g;s/_$/\.dem/'
 169     else
 170         echo ""
 171     fi
 172 }
 173
 174 _get_active_demo_workers() {
 175     if [[ $(pgrep -caf "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_") -gt 0 ]]; then
 176         pgrep -af "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_" |grep "dev/null" |awk '{ print $11"_"$17 }' |sed 's/"//g;s/_$//'
 177     else
 178         echo ""
 179     fi
 180 }
 181
 182 _get_queue_jobs() {
 183     local queue_file=$1
 184     if [[ -f ${queue_file} ]]; then
 185         cat ${queue_file} |awk '{ print $14 }'|sed 's/"//g;s/_$/\.dem/'
 186     else
 187         echo ""
 188     fi
 189 }
 190
 191 _get_completed_jobs() {
 192     if [[ -f ${LOG_FILE} ]]; then
 193         cat ${LOG_FILE} |awk '{ print $22 }'|sed 's/"//g;s/_$/\.dem/'
 194     else
 195         echo ""
 196     fi
 197 }
 198
 199 _run_compress_jobs() {
 200     local queue_file=${QUEUE_FILE_COMPRESSING}
 201     if [[ $1 ]]; then
 202         queue_file=$1
 203     fi
 204     local start=$(date +%s)
 205     (
 206         flock -n 9 || exit 99
 207         trap _cleanup_compress EXIT
 208         if [[ -f ${queue_file} ]]; then
 209             parallel -j${JOBS} --progress --eta --joblog "${LOG_FILE}" < ${queue_file}
 210         else
 211             echo "[ ERROR ] No jobs found"
 212         fi
 213     ) 9>${LOCK_FILE_COMPRESSING}
 214     if [[ $? -eq 99 ]]; then
 215         echo "[ ERROR ] lockfile exists, remove if you're sure jobs aren't running: ${LOCK_FILE_COMPRESSING}"
 216         exit 1
 217     fi
 218     local end=$(date +%s)
 219     local runtime=$((end-start))
 220     printf 'Video Compression Time: %02dh:%02dm:%02ds\n' $((runtime/3600)) $((runtime%3600/60)) $((runtime%60))
 221 }
 222
 223 _run_demo_jobs() {
 224     local queue_file=${QUEUE_FILE_DEMOS}
 225     if [[ $1 ]]; then
 226         queue_file=$1
 227     fi
 228     local start=$(date +%s)
 229     if [[ ${KILLER_KEYWORD_WATCHER} ]]; then
 230         (sleep 5 && _log_killer_keyword_watcher ${KILLER_KEYWORD}) > /dev/null 2>&1 &
 231     fi
 232     if [[ $2 == "summary" ]]; then
 233         (sleep 5 && echo && list_jobs) &
 234     fi
 235     (
 236         flock -n 9 || exit 99
 237         trap _cleanup EXIT
 238         if [[ -f ${queue_file} ]]; then
 239             parallel -j${JOBS} --progress --eta --joblog "${LOG_FILE}" < ${queue_file}
 240         else
 241             echo "[ ERROR ] No jobs found"
 242         fi
 243     ) 9>${LOCK_FILE}
 244     if [[ $? -eq 99 ]]; then
 245         echo "[ ERROR ] lockfile exists, remove if you're sure jobs aren't running: ${LOCK_FILE}"
 246         exit 1
 247     fi
 248     local end=$(date +%s)
 249     local runtime=$((end-start))
 250     printf 'Demo Encoding Time: %02dh:%02dm:%02ds\n' $((runtime/3600)) $((runtime%3600/60)) $((runtime%60))
 251 }
 252
 253 # Cleanup Helpers
 254 ##################
 255
 256 _cleanup() {
 257     rm -f ${QUEUE_FILE_DEMOS}
 258     rm -f ${LOCK_FILE}
 259     rm -f ${USERDIR}/data/*.log
 260     export KILLER_KEYWORD_WATCHING=false
 261     sleep 1
 262     _kill_xonotic
 263 }
 264
 265 _cleanup_children() {
 266     kill $(jobs -pr)
 267 }
 268
 269 _cleanup_compress() {
 270     rm -f ${QUEUE_FILE_COMPRESSING}
 271 }
 272
 273 _kill_xonotic() {
 274     pkill -f "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_"
 275 }
 276
 277 # Application Helpers
 278 ######################
 279
 280 _check_if_compress() {
 281     local compress=$1; shift
 282     local videos=$@
 283     trap _cleanup_children SIGINT SIGTERM EXIT
 284     if [[ ${compress} == "true" ]]; then
 285         _run_compress_jobs ${QUEUE_FILE_COMPRESSING}
 286     fi
 287 }
 288
 289 _log_killer_keyword_watcher() {
 290     local keyword="$@"
 291     until [[ ${KILLER_KEYWORD_WATCHING} != "true" ]]; do
 292         log_killer_keyword ${keyword}
 293         sleep ${KILLER_KEYWORD_WAIT}
 294     done
 295 }
 296
 297 # Commands
 298 ###########
 299
 300 _run_xvfb() {
 301     if [[ ! -f /tmp/.X1-lock ]]; then
 302         /usr/bin/Xvfb :1 -screen 0 ${DIMENSIONS}x16 +extension RENDER & xvfb_pid=$!
 303     else
 304         xvfb_pid=$(pgrep -f Xvfb)
 305     fi
 306     echo "[ INFO ] Xvfb PID: ${xvfb_pid}"
 307 }
 308
 309 compress() {
 310     if [[ ${FFMPEG} == "" ]]; then
 311         echo "[ ERROR ] ffmpeg or avconv required"
 312         exit 1
 313     fi
 314     if [[ ! $1 ]]; then
 315         echo "[ ERROR ] Video name required"
 316         exit 1
 317     fi
 318     local video_file=$1; shift
 319     local type="mp4"
 320     local cleanup=""
 321     if [[ $1 =~ ${REGEX_VIDEO_TYPES} ]]; then
 322         type=$2
 323         if [[ $2 == "--cleanup" ]]; then
 324             cleanup=$2
 325         fi
 326     elif [[ $1 == "--cleanup" ]]; then
 327         cleanup=$1
 328     else
 329         echo "[ ERROR ] Invalid type specified"
 330     fi
 331
 332     # compress
 333     local command=$(_get_compression_command ${USERDIR}/data/${video_file} ${type})
 334     echo ${command}
 335     echo "[ INFO ] Compressing '${video_file}'"
 336     _queue_add_compression_jobs ${QUEUE_FILE_COMPRESSING} ${type} ${video_file[@]}
 337     cat ${QUEUE_FILE_COMPRESSING}
 338     _run_compress_jobs ${QUEUE_FILE_COMPRESSING}
 339
 340     if [[ ${cleanup} == "--cleanup" ]]; then
 341         echo "[ INFO ] Cleaning up"
 342         echo rm ${video_file}
 343     fi
 344 }
 345
 346 list_jobs() {
 347     completed_jobs=$(_get_completed_jobs)
 348     active_jobs=$(_get_active_demo_jobs)
 349     all_jobs=$(_get_queue_jobs ${QUEUE_FILE_DEMOS})
 350
 351     echo -e "\nActive Jobs:\n-----------"
 352     if [[ ${active_jobs} == "" ]]; then
 353         echo "<None>"
 354     else
 355         echo ${active_jobs} |tr ' ' '\n' |sort |uniq -u
 356     fi
 357
 358     echo -e "\nQueued Jobs:\n-----------"
 359     if [[ ${#all_jobs[@]} -eq 0 ]]; then
 360         echo "<None>"
 361     else
 362         if [[ ${active_jobs} == "" ]]; then
 363             echo "<None>"
 364         else
 365             non_queued_jobs=$(echo "${active_jobs[@]}" "${completed_jobs[@]}" |tr ' ' '\n' |sort |uniq -u)
 366             queued_jobs=$(echo "${all_jobs[@]}" "${non_queued_jobs[@]}" |tr ' ' '\n' |sort |uniq -u)
 367             if [[ ${queued_jobs} == "" ]]; then
 368                 echo "<None>"
 369             else
 370                 echo ${queued_jobs} | tr ' ' '\n' | sort | uniq -u
 371             fi
 372         fi
 373     fi
 374
 375     echo -e "\nCompleted Jobs:\n--------------"
 376     if [[ ${completed_jobs} == "" ]]; then
 377         echo "<None>"
 378     else
 379         echo ${completed_jobs} | tr ' ' '\n' | sort | uniq -u
 380     fi
 381
 382     echo
 383
 384     if  [[ $1 == "-f" ]]; then
 385         sleep ${LIST_JOBS_FOLLOW_WAIT}
 386         clear
 387         date
 388         list_jobs $1
 389     fi
 390 }
 391
 392 log_completed_jobs() {
 393     local extra_flags=""
 394     if [[ $1 ]]; then
 395         extra_flags=$1
 396     fi
 397     tail ${extra_flags} ${LOG_FILE}
 398 }
 399
 400 log_killer_keyword() {
 401     local keyword="$@"
 402     local workers=$(log_keyword_grep "worker" "${keyword}")
 403     for z in ${workers[@]}; do
 404         local process=${z[0]}
 405         local pid=$(pgrep -fo ${process})
 406         echo "killing PID: ${pid} | ${process}"
 407         kill ${pid}
 408      done
 409 }
 410
 411 log_keyword_grep() {
 412     if [[ ! $2 ]]; then
 413         echo "[ ERROR ] Keyword required"
 414         exit 1
 415     fi
 416     local type=${1:-worker}; shift
 417     local keyword="$@"
 418     for worker in $(_get_active_demo_workers); do
 419         local log_file="${worker}.log"
 420         local keyword_count=$(grep -c "${keyword}" "${USERDIR}/data/${log_file}")
 421         if [[ ${keyword_count} > 0 ]]; then
 422             if [[ ${type} == "worker" ]]; then
 423                 echo "${worker}"
 424             else
 425                 echo "[ worker ] ${worker}"
 426                 grep "${keyword}" "${USERDIR}/data/${log_file}"
 427             fi
 428         fi
 429     done
 430 }
 431
 432 process_batch() {
 433     local demo_list_file=${DEFAULT_DEMO_LIST_FILE}
 434     local timeout=${JOB_TIMEOUT}
 435     local -a videos=()
 436     if [[ $1 =~ ${REGEX_DEMOLIST_FILE} ]]; then
 437         demo_list_file=$1; shift
 438     fi
 439     if [[ $1 =~ ${REGEX_DURATION} ]]; then
 440         timeout=$1; shift
 441     fi
 442     local compress=${COMPRESS}
 443     if [[ $1 == "--compress" ]]; then
 444         compress="true"
 445     fi
 446     echo "[ INFO ] Using '${demo_list_file}' with a timeout of ${timeout}"
 447     local demos=$(_get_demos_from_file ${demo_list_file})
 448     _queue_add_demo_jobs ${QUEUE_FILE_DEMOS} ${timeout} ${demos[@]}
 449     if [[ ${compress} == "true" ]]; then
 450         for v in ${demos[@]}; do
 451             videos+=("video/$(basename ${v} | sed 's/.dem$/_000.ogv/')")
 452         done
 453         _queue_add_compression_jobs ${QUEUE_FILE_COMPRESSING} "mp4" "${videos[@]}"
 454     fi
 455     _run_demo_jobs ${QUEUE_FILE_DEMOS} "summary" && \
 456         _check_if_compress ${compress} "${videos[@]}"
 457 }
 458
 459 process_single() {
 460     if [[ ! $1 ]]; then
 461         echo "[ ERROR ] Demo name required"
 462         exit 1
 463     fi
 464     local demo_file=$1
 465     local timeout=${JOB_TIMEOUT}
 466     if [[ $2 =~ ${REGEX_DURATION} ]]; then
 467         timeout=$2; shift
 468     fi
 469     local compress=${COMPRESS}
 470     if [[ $2 == "--compress" ]]; then
 471         compress="true"
 472     fi
 473     echo "[ INFO ] Using '${demo_file}' with a timeout of ${timeout}"
 474     _queue_add_demo_jobs ${QUEUE_FILE_DEMOS} ${timeout} ${demo_file}
 475     if [[ ${compress} == "true" ]]; then
 476         local video_file="video/$(basename "${demo_file}" .dem)_000.ogv"
 477         _queue_add_compression_jobs ${QUEUE_FILE_COMPRESSING} "mp4" "${video_file}"
 478     fi
 479     _run_demo_jobs ${QUEUE_FILE_DEMOS} "summary" && \
 480         _check_if_compress ${compress} ${video_file}
 481 }
 482
 483 _version() {
 484     echo ${VERSION}
 485 }
 486
 487 _help() {
 488     echo "./encode-demos.sh
 489
 490 FLAGS
 491
 492     --version                                   prints the version string
 493
 494 COMMANDS
 495
 496     Encoding
 497     --------
 498     batch  [demos.txt] [timeout] [--compress]   batch process a list of demos from file relative to \$USERDIR/data
 499     single <demo> [timeout] [--compress]        process a single demo file in \$USERDIR/data. ex: demos/cool.dem
 500                                                 'timeout' does not include '--compress', compress starts a new job
 501     Compression
 502     -----------
 503     compress <video> [mp4|webm] [--cleanup]     compress an encoded ogv in \$USERDIR/data, ex: video/cool.ogv
 504
 505     Job Management
 506     --------------
 507     grep <keyword>                              grep the server logs of the workers
 508     kkill <keyword>                             keyword kill, kill a worker if string is matched
 509     list [-f]                                   list currently active/queued/completed jobs
 510     log [-f]                                    tail the current log (-f follows log)
 511
 512 EXAMPLES
 513
 514     # outputs \$USERDIR/data/video/2015-06-11_00-26_solarium.ogv (very large)
 515     ./encode-demos.sh single demos/2015-06-11_00-26_solarium.dem
 516
 517     # outputs \$USERDIR/data/video/2015-06-11_00-26_solarium.mp4 (optimal for youtube)
 518     ./encode-demos.sh single demos/2015-06-11_00-26_solarium.dem --compress
 519
 520     # batch
 521     ./encode-demos.sh batch demos.txt --compress
 522
 523     # compress a video in \$USERDIR/data (outputs test.mp4, and deletes the original)
 524     ./encode-demos.sh compress video/test.ogv --cleanup
 525
 526     # list jobs
 527     ./encode-demos.sh list
 528
 529     # inspect worker server logs
 530     ./encode-demos.sh grep \"connected\"
 531
 532     # follow a completed job log
 533     ./encode-demos.sh log -f
 534 "
 535 }
 536
 537 case $1 in
 538     # flags
 539     '--version')        _version;;
 540     ## commands
 541     # encoding
 542     'batch')            _run_xvfb; process_batch $2 $3 $4;;
 543     'single')           _run_xvfb; process_single $2 $3 $4;;
 544     # compression
 545     'compress')         compress $2 $3 $4;;
 546     # monitoring/management
 547     'grep')             log_keyword_grep 'normal' $2;;
 548     'kkill')            log_killer_keyword $2;;
 549     'list')             list_jobs $2;;
 550     'log')              log_completed_jobs $2;;
 551     # default
 552     *)                  _help; exit 0;;
 553 esac