2 # name: encode-demos.sh
4 # author: Tyler "-z-" Mulligan <z@xnz.me>
7 # description: headless encoding of demo files to HD video concurrently with Xfvb and parallel
9 # The encoding is done with a full Xonotic client inside Xfvb.
10 # parallel is acting as a job queue.
11 # You may want to create a new userdir such as `~/.xonotic-clean` for encoding.
12 # If you don't want certain details of your player config carrying over.
14 # The following is a good starting point for 1080p videos:
22 # scr_screenshot_gammaboost 1
23 # cl_capturevideo_width 1920
24 # cl_capturevideo_height 1080
25 # cl_capturevideo_fps 60
26 # cl_capturevideo_ogg 1
27 # cl_capturevideo_ogg_theora_quality 63
28 # cl_capturevideo_ogg_theora_bitrate -1
29 # cl_capturevideo_ogg_theora_keyframe_bitrate_multiplier 2
30 # cl_capturevideo_ogg_theora_keyframe_maxinterval 500
31 # cl_capturevideo_ogg_theora_keyframe_mininterval 1
32 # cl_capturevideo_ogg_theora_keyframe_auto_threshold 80
33 # cl_capturevideo_ogg_theora_noise_sensitivity 0
34 # cl_capturevideo_ogg_vorbis_quality 10
37 # defer 5 "menu_watermark \"\""
38 # set cl_allow_uid2name 0; set cl_allow_uidtracking 0
39 # con_notify 0; con_notifysize 0; con_notifytime 0; showspeed 0; showfps 0
44 USERDIR=${HOME}/.xonotic-clean # path to Xonotic userdir for client that does encoding
45 GAMEDIR=${USERDIR}/data # path to Xonotic userdir for client that does encoding
46 XONOTIC_BIN="./all" # binary used to launch Xonotic
47 JOB_TIMEOUT="4h" # if demo doesn't quit itself or hangs
48 JOBS=4 # number of concurrent jobs
49 DEFAULT_DEMO_LIST_FILE="demos.txt" # for batch
50 DISPLAY=:1.0 # display for Xvfb
51 DIMENSIONS=1920x1080 # dimensions of virtual display
52 COMPRESS=false # whether to compress by default
53 KILLER_KEYWORD_WATCHER=true # watch server logs for keyword, kill worker if true
54 KILLER_KEYWORD="Server Disconnected" # keyword
55 KILLER_KEYWORD_WAIT="10s" # time to wait between polling watchers
56 LIST_JOBS_FOLLOW_WAIT="10s" # how often to poll the job list with -f
59 SCRIPT_NAME=$(basename $0 .sh)
60 VERSION=$(awk 'NR == 3 {print $3; exit}' $0)
61 FFMPEG=$(which ffmpeg)
62 QUEUE_FILE_DEMOS="/tmp/${SCRIPT_NAME}.jobqueue"
63 QUEUE_FILE_COMPRESSING="/tmp/${SCRIPT_NAME}_compress.jobqueue"
64 LOCK_FILE="/tmp/${SCRIPT_NAME}.lock"
65 LOCK_FILE_COMPRESSING="/tmp/${SCRIPT_NAME}.lock"
66 LOG_FILE="${SCRIPT_NAME}.log"
67 REGEX_DEMOLIST_FILE="^[[:alnum:]]+\.txt$"
68 REGEX_DURATION="^[0-9]+(d|h|m|s)$"
69 REGEX_VIDEO_TYPES="^(mp4|webm)$"
72 export KILLER_KEYWORD_WATCHING=true
76 _check_xonotic_dir() {
78 if [[ ! -d ${xon_dir} ]]; then
79 echo "[ ERROR ] Unable to locate Xonotic"; exit 1
84 relative_dir=$(dirname $0)/../..
85 _check_xonotic_dir ${relative_dir}
86 export XONOTIC_DIR="$(cd ${relative_dir}; pwd)"
90 pkill -f "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_"
96 _get_compression_command() {
97 if [[ ${FFMPEG} == "" ]]; then
98 echo "[ ERROR ] ffmpeg or avconv required"
102 echo "[ ERROR ] Video name required"
107 if [[ $2 =~ ${REGEX_VIDEO_TYPES} ]]; then
111 if [[ ${type} == "mp4" ]]; then
112 local output_video_file=$(echo ${video_file} | sed 's/\.ogv$/\.mp4/')
113 command="${FFMPEG} -i ${video_file} -y -codec:v libx264 -crf 21 -bf 2 -flags +cgop -pix_fmt yuv420p -codec:a aac -strict -2 -b:a 384k -r:a 48000 -movflags faststart ${output_video_file}"
114 elif [[ ${type} == "webm" ]]; then
115 local output_video_file=$(echo ${video_file} | sed 's/\.ogv$/\.webm/')
116 command="${FFMPEG} -i ${video_file} -y -acodec libvorbis -aq 5 -ac 2 -qmax 25 -threads 2 ${output_video_file}"
121 _get_demo_command() {
124 name_format=$(basename "${demo_file}" .dem)
125 command="${XONOTIC_DIR}/${XONOTIC_BIN} run sdl -simsound -sessionid xonotic_${SCRIPT_NAME}_${index} -userdir \"${USERDIR}\" \
126 +log_file \"xonotic_${SCRIPT_NAME}_${index}_${name_format}.log\" \
127 +cl_capturevideo_nameformat \"${name_format}_\" \
128 +cl_capturevideo_number 0 \
129 +playdemo \"${demo_file}\" \
130 +toggle cl_capturevideo \
131 +alias cl_hook_shutdown quit \
136 _get_demos_from_file() {
138 if [[ -f ${file} ]]; then
141 IFS=$'\n' read -d '' -r -a lines < ${file}
153 local nice_name=${command};
154 local nice_queue_name=${queue_file##*/};
158 echo "[ INFO ] '${nice_queue_name/.jobqueue/}' new job: ${nice_name}"
159 echo "${command}" >> ${queue_file}
162 _queue_add_compression_jobs() {
163 local queue_file=$1; shift
166 for video_file in ${videos[@]}; do
167 local command=$(_get_compression_command ${GAMEDIR}/${video_file} ${type})
168 _queue_add_job ${queue_file} "${command}" ${video_file}
172 _queue_add_demo_jobs() {
173 local queue_file=$1; shift
174 local timeout=$1; shift
177 for demo_file in ${demos[@]}; do
178 local command=$(_get_demo_command ${demo_file} ${i})
179 command="timeout ${timeout} ${command}"
180 _queue_add_job ${queue_file} "${command}" ${demo_file}
185 _get_active_demo_jobs() {
186 if [[ $(pgrep -caf "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_") -gt 0 ]]; then
187 pgrep -af "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_" |grep "dev/null" |awk '{ print $17 }' |sed 's/"//g;s/_$/\.dem/'
193 _get_active_demo_workers() {
194 if [[ $(pgrep -caf "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_") -gt 0 ]]; then
195 pgrep -af "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_" |grep "dev/null" |awk '{ print $11"_"$17 }' |sed 's/"//g;s/_$//'
203 if [[ -f ${queue_file} ]]; then
204 cat ${queue_file} |awk '{ print $14 }'|sed 's/"//g;s/_$/\.dem/'
210 _get_completed_jobs() {
211 if [[ -f ${LOG_FILE} ]]; then
212 cat ${LOG_FILE} |awk '{ print $22 }'|sed 's/"//g;s/_$/\.dem/'
218 _run_compress_jobs() {
219 local queue_file=${QUEUE_FILE_COMPRESSING}
223 local start=$(date +%s)
225 flock -n 9 || exit 99
226 trap _cleanup_compress EXIT
227 if [[ -f ${queue_file} ]]; then
228 parallel -j${JOBS} --progress --eta --joblog "${LOG_FILE}" < ${queue_file}
230 echo "[ ERROR ] No jobs found"
232 ) 9>${LOCK_FILE_COMPRESSING}
233 if [[ $? -eq 99 ]]; then
234 echo "[ ERROR ] lockfile exists, remove if you're sure jobs aren't running: ${LOCK_FILE_COMPRESSING}"
237 local end=$(date +%s)
238 local runtime=$((end-start))
239 printf 'Video Compression Time: %02dh:%02dm:%02ds\n' $((runtime/3600)) $((runtime%3600/60)) $((runtime%60))
243 local queue_file=${QUEUE_FILE_DEMOS}
247 local start=$(date +%s)
248 if [[ ${KILLER_KEYWORD_WATCHER} ]]; then
249 (sleep 5 && _log_killer_keyword_watcher ${KILLER_KEYWORD}) > /dev/null 2>&1 &
251 if [[ $2 == "summary" ]]; then
252 (sleep 5 && echo && list_jobs) &
255 flock -n 9 || exit 99
257 if [[ -f ${queue_file} ]]; then
258 parallel -j${JOBS} --progress --eta --joblog "${LOG_FILE}" < ${queue_file}
260 echo "[ ERROR ] No jobs found"
263 if [[ $? -eq 99 ]]; then
264 echo "[ ERROR ] lockfile exists, remove if you're sure jobs aren't running: ${LOCK_FILE}"
267 local end=$(date +%s)
268 local runtime=$((end-start))
269 printf 'Demo Encoding Time: %02dh:%02dm:%02ds\n' $((runtime/3600)) $((runtime%3600/60)) $((runtime%60))
276 rm -f ${QUEUE_FILE_DEMOS}
278 rm -f ${GAMEDIR}/*.log
279 export KILLER_KEYWORD_WATCHING=false
284 _cleanup_children() {
288 _cleanup_compress() {
289 rm -f ${QUEUE_FILE_COMPRESSING}
292 # Application Helpers
293 ######################
295 _check_if_compress() {
296 local compress=$1; shift
298 trap _cleanup_children SIGINT SIGTERM EXIT
299 if [[ ${compress} == "true" ]]; then
300 _run_compress_jobs ${QUEUE_FILE_COMPRESSING}
304 _log_killer_keyword_watcher() {
306 until [[ ${KILLER_KEYWORD_WATCHING} != "true" ]]; do
307 log_killer_keyword ${keyword}
308 sleep ${KILLER_KEYWORD_WAIT}
316 if [[ ! -f /tmp/.X1-lock ]]; then
317 /usr/bin/Xvfb :1 -screen 0 ${DIMENSIONS}x16 +extension RENDER & xvfb_pid=$!
319 xvfb_pid=$(pgrep -f Xvfb)
321 echo "[ INFO ] Xvfb PID: ${xvfb_pid}"
325 if [[ ${FFMPEG} == "" ]]; then
326 echo "[ ERROR ] ffmpeg or avconv required"
330 echo "[ ERROR ] Video name required"
333 local video_file=$1; shift
336 if [[ $1 =~ ${REGEX_VIDEO_TYPES} ]]; then
338 if [[ $2 == "--cleanup" ]]; then
341 elif [[ $1 == "--cleanup" ]]; then
344 echo "[ ERROR ] Invalid type specified"
348 local command=$(_get_compression_command ${GAMEDIR}/${video_file} ${type})
350 echo "[ INFO ] Compressing '${video_file}'"
351 _queue_add_compression_jobs ${QUEUE_FILE_COMPRESSING} ${type} ${video_file[@]}
352 cat ${QUEUE_FILE_COMPRESSING}
353 _run_compress_jobs ${QUEUE_FILE_COMPRESSING}
355 if [[ ${cleanup} == "--cleanup" ]]; then
356 echo "[ INFO ] Cleaning up"
357 echo rm ${video_file}
362 completed_jobs=$(_get_completed_jobs)
363 active_jobs=$(_get_active_demo_jobs)
364 all_jobs=$(_get_queue_jobs ${QUEUE_FILE_DEMOS})
366 echo -e "\nActive Jobs:\n-----------"
367 if [[ ${active_jobs} == "" ]]; then
370 echo ${active_jobs} |tr ' ' '\n' |sort |uniq -u
373 echo -e "\nQueued Jobs:\n-----------"
374 if [[ ${#all_jobs[@]} -eq 0 ]]; then
377 if [[ ${active_jobs} == "" ]]; then
380 non_queued_jobs=$(echo "${active_jobs[@]}" "${completed_jobs[@]}" |tr ' ' '\n' |sort |uniq -u)
381 queued_jobs=$(echo "${all_jobs[@]}" "${non_queued_jobs[@]}" |tr ' ' '\n' |sort |uniq -u)
382 if [[ ${queued_jobs} == "" ]]; then
385 echo ${queued_jobs} | tr ' ' '\n' | sort | uniq -u
390 echo -e "\nCompleted Jobs:\n--------------"
391 if [[ ${completed_jobs} == "" ]]; then
394 echo ${completed_jobs} | tr ' ' '\n' | sort | uniq -u
399 if [[ $1 == "-f" ]]; then
400 sleep ${LIST_JOBS_FOLLOW_WAIT}
407 log_completed_jobs() {
412 tail ${extra_flags} ${LOG_FILE}
415 log_killer_keyword() {
417 local workers=$(log_keyword_grep "worker" "${keyword}")
418 for z in ${workers[@]}; do
419 local process=${z[0]}
420 local pid=$(pgrep -fo ${process})
421 echo "killing PID: ${pid} | ${process}"
428 echo "[ ERROR ] Keyword required"
431 local type=${1:-worker}; shift
433 for worker in $(_get_active_demo_workers); do
434 local log_file="${worker}.log"
435 local keyword_count=$(grep -c "${keyword}" "${GAMEDIR}/${log_file}")
436 if [[ ${keyword_count} > 0 ]]; then
437 if [[ ${type} == "worker" ]]; then
440 echo "[ worker ] ${worker}"
441 grep "${keyword}" "${GAMEDIR}/${log_file}"
448 local demo_list_file=${DEFAULT_DEMO_LIST_FILE}
449 local timeout=${JOB_TIMEOUT}
451 if [[ $1 =~ ${REGEX_DEMOLIST_FILE} ]]; then
452 demo_list_file=$1; shift
454 if [[ $1 =~ ${REGEX_DURATION} ]]; then
457 local compress=${COMPRESS}
458 if [[ $1 == "--compress" ]]; then
461 echo "[ INFO ] Using '${demo_list_file}' with a timeout of ${timeout}"
462 local demos=$(_get_demos_from_file ${demo_list_file})
463 _queue_add_demo_jobs ${QUEUE_FILE_DEMOS} ${timeout} ${demos[@]}
464 if [[ ${compress} == "true" ]]; then
465 for v in ${demos[@]}; do
466 videos+=("video/$(basename ${v} | sed 's/.dem$/_000.ogv/')")
468 _queue_add_compression_jobs ${QUEUE_FILE_COMPRESSING} "mp4" "${videos[@]}"
470 _run_demo_jobs ${QUEUE_FILE_DEMOS} "summary" && \
471 _check_if_compress ${compress} "${videos[@]}"
476 echo "[ ERROR ] Demo name required"
480 local timeout=${JOB_TIMEOUT}
481 if [[ $2 =~ ${REGEX_DURATION} ]]; then
484 local compress=${COMPRESS}
485 if [[ $2 == "--compress" ]]; then
488 echo "[ INFO ] Using '${demo_file}' with a timeout of ${timeout}"
489 _queue_add_demo_jobs ${QUEUE_FILE_DEMOS} ${timeout} ${demo_file}
490 if [[ ${compress} == "true" ]]; then
491 local video_file="video/$(basename "${demo_file}" .dem)_000.ogv"
492 _queue_add_compression_jobs ${QUEUE_FILE_COMPRESSING} "mp4" "${video_file}"
494 _run_demo_jobs ${QUEUE_FILE_DEMOS} "summary" && \
495 _check_if_compress ${compress} ${video_file}
503 echo "./encode-demos.sh
507 --version prints the version string
513 batch [demos.txt] [timeout] [--compress] batch process a list of demos from file relative to \$USERDIR/data
514 single <demo> [timeout] [--compress] process a single demo file in \$USERDIR/data. ex: demos/cool.dem
515 'timeout' does not include '--compress', compress starts a new job
518 compress <video> [mp4|webm] [--cleanup] compress an encoded ogv in \$USERDIR/data, ex: video/cool.ogv
522 grep <keyword> grep the server logs of the workers
523 kkill <keyword> keyword kill, kill a worker if string is matched
524 list [-f] list currently active/queued/completed jobs
525 log [-f] tail the current log (-f follows log)
529 # outputs \$USERDIR/data/video/2015-06-11_00-26_solarium.ogv (very large)
530 ./encode-demos.sh single demos/2015-06-11_00-26_solarium.dem
532 # outputs \$USERDIR/data/video/2015-06-11_00-26_solarium.mp4 (optimal for youtube)
533 ./encode-demos.sh single demos/2015-06-11_00-26_solarium.dem --compress
536 ./encode-demos.sh batch demos.txt --compress
538 # compress a video in \$USERDIR/data (outputs test.mp4, and deletes the original)
539 ./encode-demos.sh compress video/test.ogv --cleanup
542 ./encode-demos.sh list
544 # inspect worker server logs
545 ./encode-demos.sh grep \"connected\"
547 # follow a completed job log
548 ./encode-demos.sh log -f
550 # Override the path to Xonotic (assumed from relative location of this script)
551 XONOTIC_DIR=\$HOME/some/other/dir; ./misc/tools/encode-demos.sh --version
558 # Allow for overriding the path assumption
559 # XONOTIC_DIR=$HOME/some/other/dir; ./misc/tools/encode-demos.sh --version
560 if [[ -z ${XONOTIC_DIR} ]]; then
563 _check_xonotic_dir ${XONOTIC_DIR}
568 '--version') _version;;
571 'batch') _run_xvfb; process_batch $2 $3 $4;;
572 'single') _run_xvfb; process_single $2 $3 $4;;
574 'compress') compress $2 $3 $4;;
575 # monitoring/management
576 'grep') log_keyword_grep 'normal' $2;;
577 'kkill') log_killer_keyword $2;;
578 'list') list_jobs $2;;
579 'log') log_completed_jobs $2;;