2 # name: encode-demos.sh
4 # author: Tyler "-z-" Mulligan <z@xnz.me>
7 # description: headless encoding of demo files to HD video concurrently with Xfvb and parallel
9 # The encoding is done with a full Xonotic client inside Xfvb.
10 # parallel is acting as a job queue.
11 # You may want to create a new userdir such as `~/.xonotic-clean` for encoding.
12 # If you don't want certain details of your player config carrying over.
14 # The following is a good starting point for 1080p videos:
22 # scr_screenshot_gammaboost 1
23 # cl_capturevideo_width 1920
24 # cl_capturevideo_height 1080
25 # cl_capturevideo_fps 60
26 # cl_capturevideo_ogg 1
27 # cl_capturevideo_ogg_theora_quality 63
28 # cl_capturevideo_ogg_theora_bitrate -1
29 # cl_capturevideo_ogg_theora_keyframe_bitrate_multiplier 2
30 # cl_capturevideo_ogg_theora_keyframe_maxinterval 500
31 # cl_capturevideo_ogg_theora_keyframe_mininterval 1
32 # cl_capturevideo_ogg_theora_keyframe_auto_threshold 80
33 # cl_capturevideo_ogg_theora_noise_sensitivity 0
34 # cl_capturevideo_ogg_vorbis_quality 10
37 # defer 5 "menu_watermark \"\""
38 # set cl_allow_uid2name 0; set cl_allow_uidtracking 0
39 # con_notify 0; con_notifysize 0; con_notifytime 0; showspeed 0; showfps 0
44 USERDIR=${HOME}/.xonotic-clean # path to Xonotic userdir for client that does encoding
45 GAMEDIR=${USERDIR}/data # path to Xonotic gamedir for client that does encoding
46 XONOTIC_BIN="./all" # binary used to launch Xonotic
47 JOB_TIMEOUT="4h" # if demo doesn't quit itself or hangs
48 JOBS=4 # number of concurrent jobs
49 DEFAULT_DEMO_LIST_FILE="demos.txt" # for batch
50 DISPLAY=:1.0 # display for Xvfb
51 DIMENSIONS=1920x1080 # dimensions of virtual display
52 COMPRESS=false # whether to compress by default
53 KILLER_KEYWORD_WATCHER=true # watch server logs for keyword, kill worker if true
54 KILLER_KEYWORD="Server Disconnected" # keyword
55 KILLER_KEYWORD_WAIT="10s" # time to wait between polling watchers
56 LIST_JOBS_FOLLOW_WAIT="10s" # how often to poll the job list with -f
59 SCRIPT_NAME=$(basename $0 .sh)
60 VERSION=$(awk 'NR == 3 {print $3; exit}' $0)
61 FFMPEG=$(which ffmpeg)
62 QUEUE_FILE_DEMOS="/tmp/${SCRIPT_NAME}.jobqueue"
63 QUEUE_FILE_COMPRESSING="/tmp/${SCRIPT_NAME}_compress.jobqueue"
64 LOCK_FILE="/tmp/${SCRIPT_NAME}.lock"
65 LOCK_FILE_COMPRESSING="/tmp/${SCRIPT_NAME}.lock"
66 LOG_FILE="${SCRIPT_NAME}.log"
67 REGEX_DEMOLIST_FILE="^[[:alnum:]]+\.txt$"
68 REGEX_DURATION="^[0-9]+(d|h|m|s)$"
69 REGEX_VIDEO_TYPES="^(mp4|webm)$"
72 export KILLER_KEYWORD_WATCHING=true
76 _check_xonotic_dir() {
78 if [[ ! -d ${xon_dir} ]]; then
79 echo "[ ERROR ] Unable to locate Xonotic"; exit 1
84 relative_dir=$(dirname $0)/../..
85 _check_xonotic_dir ${relative_dir}
86 export XONOTIC_DIR="$(cd ${relative_dir}; pwd)"
90 pkill -f "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_"
96 _get_compression_command() {
97 if [[ ${FFMPEG} == "" ]]; then
98 echo "[ ERROR ] ffmpeg or avconv required"
102 echo "[ ERROR ] Video name required"
107 if [[ $2 =~ ${REGEX_VIDEO_TYPES} ]]; then
111 if [[ ${type} == "mp4" ]]; then
112 local output_video_file=$(echo ${video_file} | sed 's/\.ogv$/\.mp4/')
113 command="${FFMPEG} -i ${video_file} -y -codec:v libx264 -crf 21 -bf 2 -flags +cgop -pix_fmt yuv420p -codec:a aac -strict -2 -b:a 384k -r:a 48000 -movflags faststart ${output_video_file}"
114 elif [[ ${type} == "webm" ]]; then
115 local output_video_file=$(echo ${video_file} | sed 's/\.ogv$/\.webm/')
116 command="${FFMPEG} -i ${video_file} -y -acodec libvorbis -aq 5 -ac 2 -qmax 25 -threads 2 ${output_video_file}"
121 _get_demo_command() {
124 name_format=$(basename "${demo_file}" .dem)
125 command="${XONOTIC_DIR}/${XONOTIC_BIN} run sdl -simsound -sessionid xonotic_${SCRIPT_NAME}_${index} -userdir \"${USERDIR}\" \
126 +log_file \"xonotic_${SCRIPT_NAME}_${index}_${name_format}.log\" \
127 +cl_capturevideo_nameformat \"${name_format}_\" \
128 +cl_capturevideo_number 0 \
129 +playdemo \"${demo_file}\" \
130 +toggle cl_capturevideo \
131 +alias cl_hook_shutdown quit \
136 _get_demos_from_file() {
138 if [[ -f ${file} ]]; then
141 IFS=$'\n' read -d '' -r -a lines < ${file}
153 local nice_name=${command};
154 local nice_queue_name=${queue_file##*/};
158 echo "[ INFO ] '${nice_queue_name/.jobqueue/}' new job: ${nice_name}"
159 echo "${command}" >> ${queue_file}
162 _queue_add_compression_jobs() {
163 local queue_file=$1; shift
166 for video_file in ${videos[@]}; do
167 local command=$(_get_compression_command ${GAMEDIR}/${video_file} ${type})
168 _queue_add_job ${queue_file} "${command}" ${video_file}
172 _queue_add_demo_jobs() {
173 local queue_file=$1; shift
174 local timeout=$1; shift
177 for demo_file in ${demos[@]}; do
178 local command=$(_get_demo_command ${demo_file} ${i})
179 command="timeout ${timeout} ${command}"
180 _queue_add_job ${queue_file} "${command}" ${demo_file}
185 _get_active_demo_jobs() {
186 if [[ $(pgrep -caf "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_") -gt 0 ]]; then
187 pgrep -af "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_" |grep "dev/null" |awk '{ print $17 }' |sed 's/"//g;s/_$/\.dem/'
193 _get_active_demo_workers() {
194 if [[ $(pgrep -caf "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_") -gt 0 ]]; then
195 pgrep -af "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_" |grep "dev/null" |awk '{ print $11"_"$17 }' |sed 's/"//g;s/_$//'
203 if [[ -f ${queue_file} ]]; then
204 cat ${queue_file} |awk '{ print $14 }'|sed 's/"//g;s/_$/\.dem/'
210 _get_completed_jobs() {
211 if [[ -f ${LOG_FILE} ]]; then
212 cat ${LOG_FILE} |awk '{ print $22 }'|sed 's/"//g;s/_$/\.dem/'
218 _run_compress_jobs() {
219 local queue_file=${QUEUE_FILE_COMPRESSING}
223 local start=$(date +%s)
225 flock -n 9 || exit 99
226 trap _cleanup_compress EXIT
227 if [[ -f ${queue_file} ]]; then
228 parallel -j${JOBS} --progress --eta --joblog "${LOG_FILE}" < ${queue_file}
230 echo "[ ERROR ] No jobs found"
232 ) 9>${LOCK_FILE_COMPRESSING}
233 if [[ $? -eq 99 ]]; then
234 echo "[ ERROR ] lockfile exists, remove if you're sure jobs aren't running: ${LOCK_FILE_COMPRESSING}"
237 local end=$(date +%s)
238 local runtime=$((end-start))
239 printf 'Video Compression Time: %02dh:%02dm:%02ds\n' $((runtime/3600)) $((runtime%3600/60)) $((runtime%60))
243 local queue_file=${QUEUE_FILE_DEMOS}
247 local start=$(date +%s)
248 if [[ ${KILLER_KEYWORD_WATCHER} ]]; then
249 (sleep 5 && _log_killer_keyword_watcher ${KILLER_KEYWORD}) > /dev/null 2>&1 &
251 if [[ $2 == "summary" ]]; then
252 (sleep 5 && echo && list_jobs) &
255 flock -n 9 || exit 99
257 if [[ -f ${queue_file} ]]; then
258 parallel -j${JOBS} --progress --eta --joblog "${LOG_FILE}" < ${queue_file}
260 echo "[ ERROR ] No jobs found"
263 if [[ $? -eq 99 ]]; then
264 echo "[ ERROR ] lockfile exists, remove if you're sure jobs aren't running: ${LOCK_FILE}"
267 local end=$(date +%s)
268 local runtime=$((end-start))
269 printf 'Demo Encoding Time: %02dh:%02dm:%02ds\n' $((runtime/3600)) $((runtime%3600/60)) $((runtime%60))
276 rm -f ${QUEUE_FILE_DEMOS}
278 rm -f ${GAMEDIR}/*.log
279 export KILLER_KEYWORD_WATCHING=false
284 _cleanup_children() {
288 _cleanup_compress() {
289 rm -f ${QUEUE_FILE_COMPRESSING}
292 # Application Helpers
293 ######################
295 _check_if_compress() {
296 local compress=$1; shift
298 trap _cleanup_children SIGINT SIGTERM EXIT
299 if [[ ${compress} == "true" ]]; then
300 _run_compress_jobs ${QUEUE_FILE_COMPRESSING}
304 _log_killer_keyword_watcher() {
306 until [[ ${KILLER_KEYWORD_WATCHING} != "true" ]]; do
307 log_killer_keyword ${keyword}
308 sleep ${KILLER_KEYWORD_WAIT}
316 if [[ ! -f /tmp/.X1-lock ]]; then
317 /usr/bin/Xvfb :1 -screen 0 ${DIMENSIONS}x16 +extension RENDER & xvfb_pid=$!
319 xvfb_pid=$(pgrep -f Xvfb)
321 echo "[ INFO ] Xvfb PID: ${xvfb_pid}"
325 if [[ ${FFMPEG} == "" ]]; then
326 echo "[ ERROR ] ffmpeg or avconv required"
330 echo "[ ERROR ] Video name required"
333 local video_file=$1; shift
336 if [[ $1 =~ ${REGEX_VIDEO_TYPES} ]]; then
338 if [[ $2 == "--cleanup" ]]; then
341 elif [[ $1 == "--cleanup" ]]; then
344 echo "[ ERROR ] Invalid type specified"
348 local command=$(_get_compression_command ${GAMEDIR}/${video_file} ${type})
350 echo "[ INFO ] Compressing '${video_file}'"
351 _queue_add_compression_jobs ${QUEUE_FILE_COMPRESSING} ${type} ${video_file[@]}
352 cat ${QUEUE_FILE_COMPRESSING}
353 _run_compress_jobs ${QUEUE_FILE_COMPRESSING}
355 if [[ ${cleanup} == "--cleanup" ]]; then
356 echo "[ INFO ] Cleaning up"
357 echo rm ${video_file}
364 local width=${3:-320}
366 local length=${5:-999999}
368 local output=$(basename ${video%.*}.gif)
370 # Generate palette for better quality
371 ${FFMPEG} -i ${GAMEDIR}/${video} -vf fps=${fps},scale=${width}:-1:flags=lanczos,palettegen ${GAMEDIR}/tmp_palette.png
373 # Generate gif using palette
374 ${FFMPEG} -i ${GAMEDIR}/${video} -i ${GAMEDIR}/tmp_palette.png -ss ${start} -t ${length} -loop ${loop} -filter_complex "fps=${fps},scale=${width}:-1:flags=lanczos[x];[x][1:v]paletteuse" ${GAMEDIR}/${output}
376 rm ${GAMEDIR}/tmp_palette.png
380 completed_jobs=$(_get_completed_jobs)
381 active_jobs=$(_get_active_demo_jobs)
382 all_jobs=$(_get_queue_jobs ${QUEUE_FILE_DEMOS})
384 echo -e "\nActive Jobs:\n-----------"
385 if [[ ${active_jobs} == "" ]]; then
388 echo ${active_jobs} |tr ' ' '\n' |sort |uniq -u
391 echo -e "\nQueued Jobs:\n-----------"
392 if [[ ${#all_jobs[@]} -eq 0 ]]; then
395 if [[ ${active_jobs} == "" ]]; then
398 non_queued_jobs=$(echo "${active_jobs[@]}" "${completed_jobs[@]}" |tr ' ' '\n' |sort |uniq -u)
399 queued_jobs=$(echo "${all_jobs[@]}" "${non_queued_jobs[@]}" |tr ' ' '\n' |sort |uniq -u)
400 if [[ ${queued_jobs} == "" ]]; then
403 echo ${queued_jobs} | tr ' ' '\n' | sort | uniq -u
408 echo -e "\nCompleted Jobs:\n--------------"
409 if [[ ${completed_jobs} == "" ]]; then
412 echo ${completed_jobs} | tr ' ' '\n' | sort | uniq -u
417 if [[ $1 == "-f" ]]; then
418 sleep ${LIST_JOBS_FOLLOW_WAIT}
425 log_completed_jobs() {
430 tail ${extra_flags} ${LOG_FILE}
433 log_killer_keyword() {
435 local workers=$(log_keyword_grep "worker" "${keyword}")
436 for z in ${workers[@]}; do
437 local process=${z[0]}
438 local pid=$(pgrep -fo ${process})
439 echo "killing PID: ${pid} | ${process}"
446 echo "[ ERROR ] Keyword required"
449 local type=${1:-worker}; shift
451 for worker in $(_get_active_demo_workers); do
452 local log_file="${worker}.log"
453 local keyword_count=$(grep -c "${keyword}" "${GAMEDIR}/${log_file}")
454 if [[ ${keyword_count} > 0 ]]; then
455 if [[ ${type} == "worker" ]]; then
458 echo "[ worker ] ${worker}"
459 grep "${keyword}" "${GAMEDIR}/${log_file}"
466 local demo_list_file=${DEFAULT_DEMO_LIST_FILE}
467 local timeout=${JOB_TIMEOUT}
469 if [[ $1 =~ ${REGEX_DEMOLIST_FILE} ]]; then
470 demo_list_file=$1; shift
472 if [[ $1 =~ ${REGEX_DURATION} ]]; then
475 local compress=${COMPRESS}
476 if [[ $1 == "--compress" ]]; then
479 echo "[ INFO ] Using '${demo_list_file}' with a timeout of ${timeout}"
480 local demos=$(_get_demos_from_file ${demo_list_file})
481 _queue_add_demo_jobs ${QUEUE_FILE_DEMOS} ${timeout} ${demos[@]}
482 if [[ ${compress} == "true" ]]; then
483 for v in ${demos[@]}; do
484 videos+=("video/$(basename ${v} | sed 's/.dem$/_000.ogv/')")
486 _queue_add_compression_jobs ${QUEUE_FILE_COMPRESSING} "mp4" "${videos[@]}"
488 _run_demo_jobs ${QUEUE_FILE_DEMOS} "summary" && \
489 _check_if_compress ${compress} "${videos[@]}"
494 echo "[ ERROR ] Demo name required"
498 local timeout=${JOB_TIMEOUT}
499 if [[ $2 =~ ${REGEX_DURATION} ]]; then
502 local compress=${COMPRESS}
503 if [[ $2 == "--compress" ]]; then
506 echo "[ INFO ] Using '${demo_file}' with a timeout of ${timeout}"
507 _queue_add_demo_jobs ${QUEUE_FILE_DEMOS} ${timeout} ${demo_file}
508 if [[ ${compress} == "true" ]]; then
509 local video_file="video/$(basename "${demo_file}" .dem)_000.ogv"
510 _queue_add_compression_jobs ${QUEUE_FILE_COMPRESSING} "mp4" "${video_file}"
512 _run_demo_jobs ${QUEUE_FILE_DEMOS} "summary" && \
513 _check_if_compress ${compress} ${video_file}
521 echo "./encode-demos.sh
525 --version prints the version string
531 batch [demos.txt] [timeout] [--compress] batch process a list of demos from file relative to \$USERDIR/data
532 single <demo> [timeout] [--compress] process a single demo file in \$USERDIR/data. ex: demos/cool.dem
533 'timeout' does not include '--compress', compress starts a new job
536 compress <video> [mp4|webm] [--cleanup] compress an encoded ogv in \$USERDIR/data, ex: video/cool.ogv
540 gif <video> [fps] [width] [start] [length] [loop] convert a video to gif in \$USERDIR/data, ex: video/cool.ogv
544 grep <keyword> grep the server logs of the workers
545 kkill <keyword> keyword kill, kill a worker if string is matched
546 list [-f] list currently active/queued/completed jobs
547 log [-f] tail the current log (-f follows log)
551 # outputs \$USERDIR/data/video/2015-06-11_00-26_solarium.ogv (very large)
552 ./encode-demos.sh single demos/2015-06-11_00-26_solarium.dem
554 # outputs \$USERDIR/data/video/2015-06-11_00-26_solarium.mp4 (optimal for youtube)
555 ./encode-demos.sh single demos/2015-06-11_00-26_solarium.dem --compress
558 ./encode-demos.sh batch demos.txt --compress
560 # compress a video in \$USERDIR/data (outputs test.mp4, and deletes the original)
561 ./encode-demos.sh compress video/test.ogv --cleanup
563 # convert video to gif (14 fps, 640 width, start at 4s, length of 4s, loop 100 times)
564 ./encode-demos.sh gif video/2017-04-01_11-53_afterslime_000.ogv 14 640 4 4 100
567 ./encode-demos.sh list
569 # inspect worker server logs
570 ./encode-demos.sh grep \"connected\"
572 # follow a completed job log
573 ./encode-demos.sh log -f
575 # Override the path to Xonotic (assumed from relative location of this script)
576 XONOTIC_DIR=\$HOME/some/other/dir; ./misc/tools/encode-demos.sh --version
583 # Allow for overriding the path assumption
584 # XONOTIC_DIR=$HOME/some/other/dir; ./misc/tools/encode-demos.sh --version
585 if [[ -z ${XONOTIC_DIR} ]]; then
588 _check_xonotic_dir ${XONOTIC_DIR}
593 '--version') _version;;
596 'batch') _run_xvfb; process_batch $2 $3 $4;;
597 'single') _run_xvfb; process_single $2 $3 $4;;
599 'compress') compress $2 $3 $4;;
601 'gif') create_gif $2 $3 $4 $5 $6 $7;;
602 # monitoring/management
603 'grep') log_keyword_grep 'normal' $2;;
604 'kkill') log_killer_keyword $2;;
605 'list') list_jobs $2;;
606 'log') log_completed_jobs $2;;