encode-demos.sh, headless parallel encoding
[xonotic/xonotic.git] / misc / tools / encode-demos.sh
1 #!/bin/bash
2 # name: encode-demos.sh
3 # version: 0.6.1
4 # author: Tyler "-z-" Mulligan
5 # license: GPL & MIT
6 # date: 24-02-2017
7 # description: headless encoding of demo files to HD video concurrently with Xfvb and parallel
8 #
9 # The encoding is done with a full Xonotic client inside Xfvb.
10 # parallel is acting as a job queue.
11 # You may want to create a new userdir such as `~/.xonotic-clean` for encoding.
12 # If you don't want certain details of your player config carrying over.
13 #
14 # The following is a good starting point for 1080p videos:
15 #
16 # ```
17 # // autoexec.cfg
18 # bgmvolume 1
19 #
20 # vid_height 1080
21 # vid_width 1920
22 # scr_screenshot_gammaboost 1
23 # cl_capturevideo_width 1920
24 # cl_capturevideo_height 1080
25 # cl_capturevideo_fps 60
26 # cl_capturevideo_ogg 1
27 # cl_capturevideo_ogg_theora_quality 63
28 # cl_capturevideo_ogg_theora_bitrate -1
29 # cl_capturevideo_ogg_theora_keyframe_bitrate_multiplier 2
30 # cl_capturevideo_ogg_theora_keyframe_maxinterval 500
31 # cl_capturevideo_ogg_theora_keyframe_mininterval 1
32 # cl_capturevideo_ogg_theora_keyframe_auto_threshold 80
33 # cl_capturevideo_ogg_theora_noise_sensitivity 0
34 # cl_capturevideo_ogg_vorbis_quality 10
35 #
36 # // HUD stuff
37 # defer 5 "menu_watermark \"\""
38 # set cl_allow_uid2name 0; set cl_allow_uidtracking 0
39 # con_notify 0; con_notifysize 0; con_notifytime 0; showspeed 0; showfps 0
40 # ```
41 #
42
43 # Customize
44 XONDIR=${HOME}/xonotic/xonotic                      # path to ./all
45 USERDIR=${HOME}/.xonotic-clean                      # path to Xonotic userdir for client that does encoding
46 XONOTIC_BIN="./all"                                 # binary used to launch Xonotic
47 JOB_TIMEOUT="1h"                                    # if demo doesn't quit itself or hangs
48 JOBS=4                                              # number of concurrent jobs
49 DEFAULT_DEMO_LIST_FILE="demos.txt"                  # for batch
50 DISPLAY=:1.0                                        # display for Xvfb
51 DIMENSIONS=1920x1080                                # dimensions of virtual display
52 COMPRESS=false                                      # whether to compress by default
53 KILLER_KEYWORD_WATCHER=true                         # watch server logs for keyword, kill worker if true
54 KILLER_KEYWORD="Server Disconnected"                # keyword
55 KILLER_KEYWORD_WAIT="10s"                           # time to wait between polling watchers
56 LIST_JOBS_FOLLOW_WAIT="10s"                         # how often to poll the job list with -f
57
58 # Internal Constants
59 SCRIPT_NAME=$(basename $0 .sh)
60 VERSION=$(awk 'NR == 3 {print $3; exit}' $0)
61 FFMPEG=$(which ffmpeg)
62 QUEUE_FILE_DEMOS="/tmp/${SCRIPT_NAME}.jobqueue"
63 QUEUE_FILE_COMPRESSING="/tmp/${SCRIPT_NAME}_compress.jobqueue"
64 LOCK_FILE="/tmp/${SCRIPT_NAME}.lock"
65 LOCK_FILE_COMPRESSING="/tmp/${SCRIPT_NAME}.lock"
66 LOG_FILE="${SCRIPT_NAME}.log"
67 REGEX_DEMOLIST_FILE="^[[:alnum:]]+\.txt$"
68 REGEX_DURATION="^[0-9]+(d|h|m|s)$"
69 REGEX_VIDEO_TYPES="^(mp4|webm)$"
70
71 # State
72 export KILLER_KEYWORD_WATCHING=true
73
74 # Data Helpers
75 ###############
76
77 _get_compression_command() {
78     if [[ ${FFMPEG} == "" ]]; then
79         echo "[ ERROR ] ffmpeg or avconv required"
80         exit 1
81     fi
82     if [[ ! $1 ]]; then
83         echo "[ ERROR ] Video name required"
84         exit 1
85     fi
86     local video_file=$1
87     local type="mp4"
88     if [[ $2 =~ ${REGEX_VIDEO_TYPES} ]]; then
89         type=$2
90     fi
91     # compress
92     if [[ ${type} == "mp4" ]]; then
93         local output_video_file=$(echo ${video_file} | sed 's/\.ogv$/\.mp4/')
94         command="${FFMPEG} -i ${video_file} -y -codec:v libx264 -crf 21 -bf 2 -flags +cgop -pix_fmt yuv420p -codec:a aac -strict -2 -b:a 384k -r:a 48000 -movflags faststart ${output_video_file}"
95     elif [[ ${type} == "webm" ]]; then
96         local output_video_file=$(echo ${video_file} | sed 's/\.ogv$/\.webm/')
97         command="${FFMPEG} -i ${video_file} -y -acodec libvorbis -aq 5 -ac 2 -qmax 25 -threads 2 ${output_video_file}"
98     fi
99     echo ${command}
100 }
101
102 _get_demo_command() {
103     local demo_file=$1
104     local index=$2
105     name_format=$(basename "${demo_file}" .dem)
106     command="${XONDIR}/${XONOTIC_BIN} run sdl -simsound -sessionid xonotic_${SCRIPT_NAME}_${index} -userdir \"${USERDIR}\" \
107         +log_file \"xonotic_${SCRIPT_NAME}_${index}_${name_format}.log\" \
108         +cl_capturevideo_nameformat \"${name_format}_\" \
109         +cl_capturevideo_number 0 \
110         +playdemo \"${demo_file}\" \
111         +toggle cl_capturevideo \
112         +alias cl_hook_shutdown quit \
113         > /dev/null 2>&1"
114     echo ${command}
115 }
116
117 _get_demos_from_file() {
118     local file=$1
119     if [[ -f ${file} ]]; then
120         local lines
121         OLD_IFS=${IFS}
122         IFS=$'\n' read -d '' -r -a lines < ${file}
123         IFS=${OLD_IFS}
124         echo ${lines[@]}
125     fi
126 }
127
128 # Queue Helpers
129 ################
130
131 _queue_add_job() {
132     local queue_file=$1;
133     local command=$2;
134     local nice_name=${command};
135     local nice_queue_name=${queue_file##*/};
136     if [[ $3 ]]; then
137         nice_name=$3
138     fi
139     echo "[ INFO ] '${nice_queue_name/.jobqueue/}' new job: ${nice_name}"
140     echo "${command}" >> ${queue_file}
141 }
142
143 _queue_add_compression_jobs() {
144     local queue_file=$1; shift
145     local type=$1; shift
146     local videos="$@"
147     for video_file in ${videos[@]}; do
148         local command=$(_get_compression_command ${USERDIR}/data/${video_file} ${type})
149         _queue_add_job ${queue_file} "${command}" ${video_file}
150     done
151 }
152
153 _queue_add_demo_jobs() {
154     local queue_file=$1; shift
155     local timeout=$1; shift
156     local demos=$@
157     local i=0
158     for demo_file in ${demos[@]}; do
159         local command=$(_get_demo_command ${demo_file} ${i})
160         command="timeout ${timeout} ${command}"
161         _queue_add_job ${queue_file} "${command}" ${demo_file}
162         ((i++))
163     done
164 }
165
166 _get_active_demo_jobs() {
167     if [[ $(pgrep -caf "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_") -gt 0 ]]; then
168         pgrep -af "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_" |grep "dev/null" |awk '{ print $17 }' |sed 's/"//g;s/_$/\.dem/'
169     else
170         echo ""
171     fi
172 }
173
174 _get_active_demo_workers() {
175     if [[ $(pgrep -caf "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_") -gt 0 ]]; then
176         pgrep -af "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_" |grep "dev/null" |awk '{ print $11"_"$17 }' |sed 's/"//g;s/_$//'
177     else
178         echo ""
179     fi
180 }
181
182 _get_queue_jobs() {
183     local queue_file=$1
184     if [[ -f ${queue_file} ]]; then
185         cat ${queue_file} |awk '{ print $14 }'|sed 's/"//g;s/_$/\.dem/'
186     else
187         echo ""
188     fi
189 }
190
191 _get_completed_jobs() {
192     if [[ -f ${LOG_FILE} ]]; then
193         cat ${LOG_FILE} |awk '{ print $22 }'|sed 's/"//g;s/_$/\.dem/'
194     else
195         echo ""
196     fi
197 }
198
199 _run_compress_jobs() {
200     local queue_file=${QUEUE_FILE_COMPRESSING}
201     if [[ $1 ]]; then
202         queue_file=$1
203     fi
204     local start=$(date +%s)
205     (
206         flock -n 9 || exit 99
207         trap _cleanup_compress EXIT
208         if [[ -f ${queue_file} ]]; then
209             parallel -j${JOBS} --progress --eta --joblog "${LOG_FILE}" < ${queue_file}
210         else
211             echo "[ ERROR ] No jobs found"
212         fi
213     ) 9>${LOCK_FILE_COMPRESSING}
214     if [[ $? -eq 99 ]]; then
215         echo "[ ERROR ] lockfile exists, remove if you're sure jobs aren't running: ${LOCK_FILE_COMPRESSING}"
216         exit 1
217     fi
218     local end=$(date +%s)
219     local runtime=$((end-start))
220     printf 'Video Compression Time: %02dh:%02dm:%02ds\n' $((runtime/3600)) $((runtime%3600/60)) $((runtime%60))
221 }
222
223 _run_demo_jobs() {
224     local queue_file=${QUEUE_FILE_DEMOS}
225     if [[ $1 ]]; then
226         queue_file=$1
227     fi
228     local start=$(date +%s)
229     if [[ ${KILLER_KEYWORD_WATCHER} ]]; then
230         (sleep 5 && _log_killer_keyword_watcher ${KILLER_KEYWORD}) > /dev/null 2>&1 &
231     fi
232     if [[ $2 == "summary" ]]; then
233         (sleep 5 && echo && list_jobs) &
234     fi
235     (
236         flock -n 9 || exit 99
237         trap _cleanup EXIT
238         if [[ -f ${queue_file} ]]; then
239             parallel -j${JOBS} --progress --eta --joblog "${LOG_FILE}" < ${queue_file}
240         else
241             echo "[ ERROR ] No jobs found"
242         fi
243     ) 9>${LOCK_FILE}
244     if [[ $? -eq 99 ]]; then
245         echo "[ ERROR ] lockfile exists, remove if you're sure jobs aren't running: ${LOCK_FILE}"
246         exit 1
247     fi
248     local end=$(date +%s)
249     local runtime=$((end-start))
250     printf 'Demo Encoding Time: %02dh:%02dm:%02ds\n' $((runtime/3600)) $((runtime%3600/60)) $((runtime%60))
251 }
252
253 # Cleanup Helpers
254 ##################
255
256 _cleanup() {
257     rm -f ${QUEUE_FILE_DEMOS}
258     rm -f ${LOCK_FILE}
259     rm -f ${USERDIR}/data/*.log
260     export KILLER_KEYWORD_WATCHING=false
261     sleep 1
262     _kill_xonotic
263 }
264
265 _cleanup_children() {
266     kill $(jobs -pr)
267 }
268
269 _cleanup_compress() {
270     rm -f ${QUEUE_FILE_COMPRESSING}
271 }
272
273 _kill_xonotic() {
274     pkill -f "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_"
275 }
276
277 # Application Helpers
278 ######################
279
280 _check_if_compress() {
281     local compress=$1; shift
282     local videos=$@
283     trap _cleanup_children SIGINT SIGTERM EXIT
284     if [[ ${compress} == "true" ]]; then
285         _run_compress_jobs ${QUEUE_FILE_COMPRESSING}
286     fi
287 }
288
289 _log_killer_keyword_watcher() {
290     local keyword="$@"
291     until [[ ${KILLER_KEYWORD_WATCHING} != "true" ]]; do
292         log_killer_keyword ${keyword}
293         sleep ${KILLER_KEYWORD_WAIT}
294     done
295 }
296
297 # Commands
298 ###########
299
300 _run_xvfb() {
301     if [[ ! -f /tmp/.X1-lock ]]; then
302         /usr/bin/Xvfb :1 -screen 0 ${DIMENSIONS}x16 +extension RENDER & xvfb_pid=$!
303     else
304         xvfb_pid=$(pgrep -f Xvfb)
305     fi
306     echo "[ INFO ] Xvfb PID: ${xvfb_pid}"
307 }
308
309 compress() {
310     if [[ ${FFMPEG} == "" ]]; then
311         echo "[ ERROR ] ffmpeg or avconv required"
312         exit 1
313     fi
314     if [[ ! $1 ]]; then
315         echo "[ ERROR ] Video name required"
316         exit 1
317     fi
318     local video_file=$1; shift
319     local type="mp4"
320     local cleanup=""
321     if [[ $1 =~ ${REGEX_VIDEO_TYPES} ]]; then
322         type=$2
323         if [[ $2 == "--cleanup" ]]; then
324             cleanup=$2
325         fi
326     elif [[ $1 == "--cleanup" ]]; then
327         cleanup=$1
328     else
329         echo "[ ERROR ] Invalid type specified"
330     fi
331
332     # compress
333     local command=$(_get_compression_command ${USERDIR}/data/${video_file} ${type})
334     echo ${command}
335     echo "[ INFO ] Compressing '${video_file}'"
336     _queue_add_compression_jobs ${QUEUE_FILE_COMPRESSING} ${type} ${video_file[@]}
337     cat ${QUEUE_FILE_COMPRESSING}
338     _run_compress_jobs ${QUEUE_FILE_COMPRESSING}
339
340     if [[ ${cleanup} == "--cleanup" ]]; then
341         echo "[ INFO ] Cleaning up"
342         echo rm ${video_file}
343     fi
344 }
345
346 list_jobs() {
347     completed_jobs=$(_get_completed_jobs)
348     active_jobs=$(_get_active_demo_jobs)
349     all_jobs=$(_get_queue_jobs ${QUEUE_FILE_DEMOS})
350
351     echo -e "\nActive Jobs:\n-----------"
352     if [[ ${active_jobs} == "" ]]; then
353         echo "<None>"
354     else
355         echo ${active_jobs} |tr ' ' '\n' |sort |uniq -u
356     fi
357
358     echo -e "\nQueued Jobs:\n-----------"
359     if [[ ${#all_jobs[@]} -eq 0 ]]; then
360         echo "<None>"
361     else
362         if [[ ${active_jobs} == "" ]]; then
363             echo "<None>"
364         else
365             non_queued_jobs=$(echo "${active_jobs[@]}" "${completed_jobs[@]}" |tr ' ' '\n' |sort |uniq -u)
366             queued_jobs=$(echo "${all_jobs[@]}" "${non_queued_jobs[@]}" |tr ' ' '\n' |sort |uniq -u)
367             if [[ ${queued_jobs} == "" ]]; then
368                 echo "<None>"
369             else
370                 echo ${queued_jobs} | tr ' ' '\n' | sort | uniq -u
371             fi
372         fi
373     fi
374
375     echo -e "\nCompleted Jobs:\n--------------"
376     if [[ ${completed_jobs} == "" ]]; then
377         echo "<None>"
378     else
379         echo ${completed_jobs} | tr ' ' '\n' | sort | uniq -u
380     fi
381
382     echo
383
384     if  [[ $1 == "-f" ]]; then
385         sleep ${LIST_JOBS_FOLLOW_WAIT}
386         clear
387         date
388         list_jobs $1
389     fi
390 }
391
392 log_completed_jobs() {
393     local extra_flags=""
394     if [[ $1 ]]; then
395         extra_flags=$1
396     fi
397     tail ${extra_flags} ${LOG_FILE}
398 }
399
400 log_killer_keyword() {
401     local keyword="$@"
402     local workers=$(log_keyword_grep "worker" "${keyword}")
403     for z in ${workers[@]}; do
404         local process=${z[0]}
405         local pid=$(pgrep -fo ${process})
406         echo "killing PID: ${pid} | ${process}"
407         kill ${pid}
408      done
409 }
410
411 log_keyword_grep() {
412     if [[ ! $2 ]]; then
413         echo "[ ERROR ] Keyword required"
414         exit 1
415     fi
416     local type=${1:-worker}; shift
417     local keyword="$@"
418     for worker in $(_get_active_demo_workers); do
419         local log_file="${worker}.log"
420         local keyword_count=$(grep -c "${keyword}" "${USERDIR}/data/${log_file}")
421         if [[ ${keyword_count} > 0 ]]; then
422             if [[ ${type} == "worker" ]]; then
423                 echo "${worker}"
424             else
425                 echo "[ worker ] ${worker}"
426                 grep "${keyword}" "${USERDIR}/data/${log_file}"
427             fi
428         fi
429     done
430 }
431
432 process_batch() {
433     local demo_list_file=${DEFAULT_DEMO_LIST_FILE}
434     local timeout=${JOB_TIMEOUT}
435     local -a videos=()
436     if [[ $1 =~ ${REGEX_DEMOLIST_FILE} ]]; then
437         demo_list_file=$1; shift
438     fi
439     if [[ $1 =~ ${REGEX_DURATION} ]]; then
440         timeout=$1; shift
441     fi
442     local compress=${COMPRESS}
443     if [[ $1 == "--compress" ]]; then
444         compress="true"
445     fi
446     echo "[ INFO ] Using '${demo_list_file}' with a timeout of ${timeout}"
447     local demos=$(_get_demos_from_file ${demo_list_file})
448     _queue_add_demo_jobs ${QUEUE_FILE_DEMOS} ${timeout} ${demos[@]}
449     if [[ ${compress} == "true" ]]; then
450         for v in ${demos[@]}; do
451             videos+=("video/$(basename ${v} | sed 's/.dem$/_000.ogv/')")
452         done
453         _queue_add_compression_jobs ${QUEUE_FILE_COMPRESSING} "mp4" "${videos[@]}"
454     fi
455     _run_demo_jobs ${QUEUE_FILE_DEMOS} "summary" && \
456         _check_if_compress ${compress} "${videos[@]}"
457 }
458
459 process_single() {
460     if [[ ! $1 ]]; then
461         echo "[ ERROR ] Demo name required"
462         exit 1
463     fi
464     local demo_file=$1
465     local timeout=${JOB_TIMEOUT}
466     if [[ $2 =~ ${REGEX_DURATION} ]]; then
467         timeout=$2; shift
468     fi
469     local compress=${COMPRESS}
470     if [[ $2 == "--compress" ]]; then
471         compress="true"
472     fi
473     echo "[ INFO ] Using '${demo_file}' with a timeout of ${timeout}"
474     _queue_add_demo_jobs ${QUEUE_FILE_DEMOS} ${timeout} ${demo_file}
475     if [[ ${compress} == "true" ]]; then
476         local video_file="video/$(basename "${demo_file}" .dem)_000.ogv"
477         _queue_add_compression_jobs ${QUEUE_FILE_COMPRESSING} "mp4" "${video_file}"
478     fi
479     _run_demo_jobs ${QUEUE_FILE_DEMOS} "summary" && \
480         _check_if_compress ${compress} ${video_file}
481 }
482
483 _version() {
484     echo ${VERSION}
485 }
486
487 _help() {
488     echo "./encode-demos.sh
489
490 FLAGS
491
492     --version                                   prints the version string
493
494 COMMANDS
495
496     Encoding
497     --------
498     batch  [demos.txt] [timeout] [--compress]   batch process a list of demos from file relative to \$USERDIR/data
499     single <demo> [timeout] [--compress]        process a single demo file in \$USERDIR/data. ex: demos/cool.dem
500                                                 'timeout' does not include '--compress', compress starts a new job
501     Compression
502     -----------
503     compress <video> [mp4|webm] [--cleanup]     compress an encoded ogv in \$USERDIR/data, ex: video/cool.ogv
504
505     Job Management
506     --------------
507     grep <keyword>                              grep the server logs of the workers
508     kkill <keyword>                             keyword kill, kill a worker if string is matched
509     list [-f]                                   list currently active/queued/completed jobs
510     log [-f]                                    tail the current log (-f follows log)
511
512 EXAMPLES
513
514     # outputs \$USERDIR/data/video/2015-06-11_00-26_solarium.ogv (very large)
515     ./encode-demos.sh single demos/2015-06-11_00-26_solarium.dem
516
517     # outputs \$USERDIR/data/video/2015-06-11_00-26_solarium.mp4 (optimal for youtube)
518     ./encode-demos.sh single demos/2015-06-11_00-26_solarium.dem --compress
519
520     # batch
521     ./encode-demos.sh batch demos.txt --compress
522
523     # compress a video in \$USERDIR/data (outputs test.mp4, and deletes the original)
524     ./encode-demos.sh compress video/test.ogv --cleanup
525
526     # list jobs
527     ./encode-demos.sh list
528
529     # inspect worker server logs
530     ./encode-demos.sh grep \"connected\"
531
532     # follow a completed job log
533     ./encode-demos.sh log -f
534 "
535 }
536
537 case $1 in
538     # flags
539     '--version')        _version;;
540     ## commands
541     # encoding
542     'batch')            _run_xvfb; process_batch $2 $3 $4;;
543     'single')           _run_xvfb; process_single $2 $3 $4;;
544     # compression
545     'compress')         compress $2 $3 $4;;
546     # monitoring/management
547     'grep')             log_keyword_grep 'normal' $2;;
548     'kkill')            log_killer_keyword $2;;
549     'list')             list_jobs $2;;
550     'log')              log_completed_jobs $2;;
551     # default
552     *)                  _help; exit 0;;
553 esac