]> git.xonotic.org Git - xonotic/xonotic.git/blob - misc/tools/encode-demos.sh
libvorbis-0.dll - 1.3.6
[xonotic/xonotic.git] / misc / tools / encode-demos.sh
1 #!/bin/bash
2 # name: encode-demos.sh
3 # version: 0.6.3
4 # author: Tyler "-z-" Mulligan <z@xnz.me>
5 # license: GPL & MIT
6 # date: 01-04-2017
7 # description: headless encoding of demo files to HD video concurrently with Xfvb and parallel
8 #
9 # The encoding is done with a full Xonotic client inside Xfvb.
10 # parallel is acting as a job queue.
11 # You may want to create a new userdir such as `~/.xonotic-clean` for encoding.
12 # If you don't want certain details of your player config carrying over.
13 #
14 # The following is a good starting point for 1080p videos:
15 #
16 # ```
17 # // autoexec.cfg
18 # bgmvolume 1
19 #
20 # vid_height 1080
21 # vid_width 1920
22 # scr_screenshot_gammaboost 1
23 # cl_capturevideo_width 1920
24 # cl_capturevideo_height 1080
25 # cl_capturevideo_fps 60
26 # cl_capturevideo_ogg 1
27 # cl_capturevideo_ogg_theora_quality 63
28 # cl_capturevideo_ogg_theora_bitrate -1
29 # cl_capturevideo_ogg_theora_keyframe_bitrate_multiplier 2
30 # cl_capturevideo_ogg_theora_keyframe_maxinterval 500
31 # cl_capturevideo_ogg_theora_keyframe_mininterval 1
32 # cl_capturevideo_ogg_theora_keyframe_auto_threshold 80
33 # cl_capturevideo_ogg_theora_noise_sensitivity 0
34 # cl_capturevideo_ogg_vorbis_quality 10
35 #
36 # // HUD stuff
37 # defer 5 "menu_watermark \"\""
38 # set cl_allow_uid2name 0; set cl_allow_uidtracking 0
39 # con_notify 0; con_notifysize 0; con_notifytime 0; showspeed 0; showfps 0
40 # ```
41 #
42
43 # Customize
44 USERDIR=${HOME}/.xonotic-clean                      # path to Xonotic userdir for client that does encoding
45 GAMEDIR=${USERDIR}/data                             # path to Xonotic gamedir for client that does encoding
46 XONOTIC_BIN="./all"                                 # binary used to launch Xonotic
47 JOB_TIMEOUT="4h"                                    # if demo doesn't quit itself or hangs
48 JOBS=4                                              # number of concurrent jobs
49 DEFAULT_DEMO_LIST_FILE="demos.txt"                  # for batch
50 DISPLAY=:1.0                                        # display for Xvfb
51 DIMENSIONS=1920x1080                                # dimensions of virtual display
52 COMPRESS=false                                      # whether to compress by default
53 KILLER_KEYWORD_WATCHER=true                         # watch server logs for keyword, kill worker if true
54 KILLER_KEYWORD="Server Disconnected"                # keyword
55 KILLER_KEYWORD_WAIT="10s"                           # time to wait between polling watchers
56 LIST_JOBS_FOLLOW_WAIT="10s"                         # how often to poll the job list with -f
57
58 # Internal Constants
59 SCRIPT_NAME=$(basename $0 .sh)
60 VERSION=$(awk 'NR == 3 {print $3; exit}' $0)
61 FFMPEG=$(which ffmpeg)
62 QUEUE_FILE_DEMOS="/tmp/${SCRIPT_NAME}.jobqueue"
63 QUEUE_FILE_COMPRESSING="/tmp/${SCRIPT_NAME}_compress.jobqueue"
64 LOCK_FILE="/tmp/${SCRIPT_NAME}.lock"
65 LOCK_FILE_COMPRESSING="/tmp/${SCRIPT_NAME}.lock"
66 LOG_FILE="${SCRIPT_NAME}.log"
67 REGEX_DEMOLIST_FILE="^[[:alnum:]]+\.txt$"
68 REGEX_DURATION="^[0-9]+(d|h|m|s)$"
69 REGEX_VIDEO_TYPES="^(mp4|webm)$"
70
71 # State
72 export KILLER_KEYWORD_WATCHING=true
73
74 # Xonotic Helpers
75
76 _check_xonotic_dir() {
77     local xon_dir=$1
78     if [[ ! -d ${xon_dir} ]]; then
79         echo "[ ERROR ] Unable to locate Xonotic"; exit 1
80     fi
81 }
82
83 _get_xonotic_dir() {
84     relative_dir=$(dirname $0)/../..
85     _check_xonotic_dir ${relative_dir}
86     export XONOTIC_DIR="$(cd ${relative_dir}; pwd)"
87 }
88
89 _kill_xonotic() {
90     pkill -f "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_"
91 }
92
93 # Data Helpers
94 ###############
95
96 _get_compression_command() {
97     if [[ ${FFMPEG} == "" ]]; then
98         echo "[ ERROR ] ffmpeg or avconv required"
99         exit 1
100     fi
101     if [[ ! $1 ]]; then
102         echo "[ ERROR ] Video name required"
103         exit 1
104     fi
105     local video_file=$1
106     local type="mp4"
107     if [[ $2 =~ ${REGEX_VIDEO_TYPES} ]]; then
108         type=$2
109     fi
110     # compress
111     if [[ ${type} == "mp4" ]]; then
112         local output_video_file=$(echo ${video_file} | sed 's/\.ogv$/\.mp4/')
113         command="${FFMPEG} -i ${video_file} -y -codec:v libx264 -crf 21 -bf 2 -flags +cgop -pix_fmt yuv420p -codec:a aac -strict -2 -b:a 384k -r:a 48000 -movflags faststart ${output_video_file}"
114     elif [[ ${type} == "webm" ]]; then
115         local output_video_file=$(echo ${video_file} | sed 's/\.ogv$/\.webm/')
116         command="${FFMPEG} -i ${video_file} -y -acodec libvorbis -aq 5 -ac 2 -qmax 25 -threads 2 ${output_video_file}"
117     fi
118     echo ${command}
119 }
120
121 _get_demo_command() {
122     local demo_file=$1
123     local index=$2
124     name_format=$(basename "${demo_file}" .dem)
125     command="${XONOTIC_DIR}/${XONOTIC_BIN} run sdl -simsound -sessionid xonotic_${SCRIPT_NAME}_${index} -userdir \"${USERDIR}\" \
126         +log_file \"xonotic_${SCRIPT_NAME}_${index}_${name_format}.log\" \
127         +cl_capturevideo_nameformat \"${name_format}_\" \
128         +cl_capturevideo_number 0 \
129         +playdemo \"${demo_file}\" \
130         +toggle cl_capturevideo \
131         +alias cl_hook_shutdown quit \
132         > /dev/null 2>&1"
133     echo ${command}
134 }
135
136 _get_demos_from_file() {
137     local file=$1
138     if [[ -f ${file} ]]; then
139         local lines
140         OLD_IFS=${IFS}
141         IFS=$'\n' read -d '' -r -a lines < ${file}
142         IFS=${OLD_IFS}
143         echo ${lines[@]}
144     fi
145 }
146
147 # Queue Helpers
148 ################
149
150 _queue_add_job() {
151     local queue_file=$1;
152     local command=$2;
153     local nice_name=${command};
154     local nice_queue_name=${queue_file##*/};
155     if [[ $3 ]]; then
156         nice_name=$3
157     fi
158     echo "[ INFO ] '${nice_queue_name/.jobqueue/}' new job: ${nice_name}"
159     echo "${command}" >> ${queue_file}
160 }
161
162 _queue_add_compression_jobs() {
163     local queue_file=$1; shift
164     local type=$1; shift
165     local videos="$@"
166     for video_file in ${videos[@]}; do
167         local command=$(_get_compression_command ${GAMEDIR}/${video_file} ${type})
168         _queue_add_job ${queue_file} "${command}" ${video_file}
169     done
170 }
171
172 _queue_add_demo_jobs() {
173     local queue_file=$1; shift
174     local timeout=$1; shift
175     local demos=$@
176     local i=0
177     for demo_file in ${demos[@]}; do
178         local command=$(_get_demo_command ${demo_file} ${i})
179         command="timeout ${timeout} ${command}"
180         _queue_add_job ${queue_file} "${command}" ${demo_file}
181         ((i++))
182     done
183 }
184
185 _get_active_demo_jobs() {
186     if [[ $(pgrep -caf "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_") -gt 0 ]]; then
187         pgrep -af "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_" |grep "dev/null" |awk '{ print $17 }' |sed 's/"//g;s/_$/\.dem/'
188     else
189         echo ""
190     fi
191 }
192
193 _get_active_demo_workers() {
194     if [[ $(pgrep -caf "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_") -gt 0 ]]; then
195         pgrep -af "\-simsound \-sessionid xonotic_${SCRIPT_NAME}_" |grep "dev/null" |awk '{ print $11"_"$17 }' |sed 's/"//g;s/_$//'
196     else
197         echo ""
198     fi
199 }
200
201 _get_queue_jobs() {
202     local queue_file=$1
203     if [[ -f ${queue_file} ]]; then
204         cat ${queue_file} |awk '{ print $14 }'|sed 's/"//g;s/_$/\.dem/'
205     else
206         echo ""
207     fi
208 }
209
210 _get_completed_jobs() {
211     if [[ -f ${LOG_FILE} ]]; then
212         cat ${LOG_FILE} |awk '{ print $22 }'|sed 's/"//g;s/_$/\.dem/'
213     else
214         echo ""
215     fi
216 }
217
218 _run_compress_jobs() {
219     local queue_file=${QUEUE_FILE_COMPRESSING}
220     if [[ $1 ]]; then
221         queue_file=$1
222     fi
223     local start=$(date +%s)
224     (
225         flock -n 9 || exit 99
226         trap _cleanup_compress EXIT
227         if [[ -f ${queue_file} ]]; then
228             parallel -j${JOBS} --progress --eta --joblog "${LOG_FILE}" < ${queue_file}
229         else
230             echo "[ ERROR ] No jobs found"
231         fi
232     ) 9>${LOCK_FILE_COMPRESSING}
233     if [[ $? -eq 99 ]]; then
234         echo "[ ERROR ] lockfile exists, remove if you're sure jobs aren't running: ${LOCK_FILE_COMPRESSING}"
235         exit 1
236     fi
237     local end=$(date +%s)
238     local runtime=$((end-start))
239     printf 'Video Compression Time: %02dh:%02dm:%02ds\n' $((runtime/3600)) $((runtime%3600/60)) $((runtime%60))
240 }
241
242 _run_demo_jobs() {
243     local queue_file=${QUEUE_FILE_DEMOS}
244     if [[ $1 ]]; then
245         queue_file=$1
246     fi
247     local start=$(date +%s)
248     if [[ ${KILLER_KEYWORD_WATCHER} ]]; then
249         (sleep 5 && _log_killer_keyword_watcher ${KILLER_KEYWORD}) > /dev/null 2>&1 &
250     fi
251     if [[ $2 == "summary" ]]; then
252         (sleep 5 && echo && list_jobs) &
253     fi
254     (
255         flock -n 9 || exit 99
256         trap _cleanup EXIT
257         if [[ -f ${queue_file} ]]; then
258             parallel -j${JOBS} --progress --eta --joblog "${LOG_FILE}" < ${queue_file}
259         else
260             echo "[ ERROR ] No jobs found"
261         fi
262     ) 9>${LOCK_FILE}
263     if [[ $? -eq 99 ]]; then
264         echo "[ ERROR ] lockfile exists, remove if you're sure jobs aren't running: ${LOCK_FILE}"
265         exit 1
266     fi
267     local end=$(date +%s)
268     local runtime=$((end-start))
269     printf 'Demo Encoding Time: %02dh:%02dm:%02ds\n' $((runtime/3600)) $((runtime%3600/60)) $((runtime%60))
270 }
271
272 # Cleanup Helpers
273 ##################
274
275 _cleanup() {
276     rm -f ${QUEUE_FILE_DEMOS}
277     rm -f ${LOCK_FILE}
278     rm -f ${GAMEDIR}/*.log
279     export KILLER_KEYWORD_WATCHING=false
280     sleep 1
281     _kill_xonotic
282 }
283
284 _cleanup_children() {
285     kill $(jobs -pr)
286 }
287
288 _cleanup_compress() {
289     rm -f ${QUEUE_FILE_COMPRESSING}
290 }
291
292 # Application Helpers
293 ######################
294
295 _check_if_compress() {
296     local compress=$1; shift
297     local videos=$@
298     trap _cleanup_children SIGINT SIGTERM EXIT
299     if [[ ${compress} == "true" ]]; then
300         _run_compress_jobs ${QUEUE_FILE_COMPRESSING}
301     fi
302 }
303
304 _log_killer_keyword_watcher() {
305     local keyword="$@"
306     until [[ ${KILLER_KEYWORD_WATCHING} != "true" ]]; do
307         log_killer_keyword ${keyword}
308         sleep ${KILLER_KEYWORD_WAIT}
309     done
310 }
311
312 # Commands
313 ###########
314
315 _run_xvfb() {
316     if [[ ! -f /tmp/.X1-lock ]]; then
317         /usr/bin/Xvfb :1 -screen 0 ${DIMENSIONS}x16 +extension RENDER & xvfb_pid=$!
318     else
319         xvfb_pid=$(pgrep -f Xvfb)
320     fi
321     echo "[ INFO ] Xvfb PID: ${xvfb_pid}"
322 }
323
324 compress() {
325     if [[ ${FFMPEG} == "" ]]; then
326         echo "[ ERROR ] ffmpeg or avconv required"
327         exit 1
328     fi
329     if [[ ! $1 ]]; then
330         echo "[ ERROR ] Video name required"
331         exit 1
332     fi
333     local video_file=$1; shift
334     local type="mp4"
335     local cleanup=""
336     if [[ $1 =~ ${REGEX_VIDEO_TYPES} ]]; then
337         type=$2
338         if [[ $2 == "--cleanup" ]]; then
339             cleanup=$2
340         fi
341     elif [[ $1 == "--cleanup" ]]; then
342         cleanup=$1
343     else
344         echo "[ ERROR ] Invalid type specified"
345     fi
346
347     # compress
348     local command=$(_get_compression_command ${GAMEDIR}/${video_file} ${type})
349     echo ${command}
350     echo "[ INFO ] Compressing '${video_file}'"
351     _queue_add_compression_jobs ${QUEUE_FILE_COMPRESSING} ${type} ${video_file[@]}
352     cat ${QUEUE_FILE_COMPRESSING}
353     _run_compress_jobs ${QUEUE_FILE_COMPRESSING}
354
355     if [[ ${cleanup} == "--cleanup" ]]; then
356         echo "[ INFO ] Cleaning up"
357         echo rm ${video_file}
358     fi
359 }
360
361 create_gif() {
362     local video=$1
363     local fps=${2:-15}
364     local width=${3:-320}
365     local start=${4:-0}
366     local length=${5:-999999}
367     local loop=${6:-1}
368     local output=$(basename ${video%.*}.gif)
369
370     # Generate palette for better quality
371     ${FFMPEG} -i ${GAMEDIR}/${video} -vf fps=${fps},scale=${width}:-1:flags=lanczos,palettegen ${GAMEDIR}/tmp_palette.png
372
373     # Generate gif using palette
374     ${FFMPEG} -i ${GAMEDIR}/${video} -i ${GAMEDIR}/tmp_palette.png -ss ${start} -t ${length} -loop ${loop} -filter_complex "fps=${fps},scale=${width}:-1:flags=lanczos[x];[x][1:v]paletteuse" ${GAMEDIR}/${output}
375
376     rm ${GAMEDIR}/tmp_palette.png
377 }
378
379 list_jobs() {
380     completed_jobs=$(_get_completed_jobs)
381     active_jobs=$(_get_active_demo_jobs)
382     all_jobs=$(_get_queue_jobs ${QUEUE_FILE_DEMOS})
383
384     echo -e "\nActive Jobs:\n-----------"
385     if [[ ${active_jobs} == "" ]]; then
386         echo "<None>"
387     else
388         echo ${active_jobs} |tr ' ' '\n' |sort |uniq -u
389     fi
390
391     echo -e "\nQueued Jobs:\n-----------"
392     if [[ ${#all_jobs[@]} -eq 0 ]]; then
393         echo "<None>"
394     else
395         if [[ ${active_jobs} == "" ]]; then
396             echo "<None>"
397         else
398             non_queued_jobs=$(echo "${active_jobs[@]}" "${completed_jobs[@]}" |tr ' ' '\n' |sort |uniq -u)
399             queued_jobs=$(echo "${all_jobs[@]}" "${non_queued_jobs[@]}" |tr ' ' '\n' |sort |uniq -u)
400             if [[ ${queued_jobs} == "" ]]; then
401                 echo "<None>"
402             else
403                 echo ${queued_jobs} | tr ' ' '\n' | sort | uniq -u
404             fi
405         fi
406     fi
407
408     echo -e "\nCompleted Jobs:\n--------------"
409     if [[ ${completed_jobs} == "" ]]; then
410         echo "<None>"
411     else
412         echo ${completed_jobs} | tr ' ' '\n' | sort | uniq -u
413     fi
414
415     echo
416
417     if  [[ $1 == "-f" ]]; then
418         sleep ${LIST_JOBS_FOLLOW_WAIT}
419         clear
420         date
421         list_jobs $1
422     fi
423 }
424
425 log_completed_jobs() {
426     local extra_flags=""
427     if [[ $1 ]]; then
428         extra_flags=$1
429     fi
430     tail ${extra_flags} ${LOG_FILE}
431 }
432
433 log_killer_keyword() {
434     local keyword="$@"
435     local workers=$(log_keyword_grep "worker" "${keyword}")
436     for z in ${workers[@]}; do
437         local process=${z[0]}
438         local pid=$(pgrep -fo ${process})
439         echo "killing PID: ${pid} | ${process}"
440         kill ${pid}
441      done
442 }
443
444 log_keyword_grep() {
445     if [[ ! $2 ]]; then
446         echo "[ ERROR ] Keyword required"
447         exit 1
448     fi
449     local type=${1:-worker}; shift
450     local keyword="$@"
451     for worker in $(_get_active_demo_workers); do
452         local log_file="${worker}.log"
453         local keyword_count=$(grep -c "${keyword}" "${GAMEDIR}/${log_file}")
454         if [[ ${keyword_count} > 0 ]]; then
455             if [[ ${type} == "worker" ]]; then
456                 echo "${worker}"
457             else
458                 echo "[ worker ] ${worker}"
459                 grep "${keyword}" "${GAMEDIR}/${log_file}"
460             fi
461         fi
462     done
463 }
464
465 process_batch() {
466     local demo_list_file=${DEFAULT_DEMO_LIST_FILE}
467     local timeout=${JOB_TIMEOUT}
468     local -a videos=()
469     if [[ $1 =~ ${REGEX_DEMOLIST_FILE} ]]; then
470         demo_list_file=$1; shift
471     fi
472     if [[ $1 =~ ${REGEX_DURATION} ]]; then
473         timeout=$1; shift
474     fi
475     local compress=${COMPRESS}
476     if [[ $1 == "--compress" ]]; then
477         compress="true"
478     fi
479     echo "[ INFO ] Using '${demo_list_file}' with a timeout of ${timeout}"
480     local demos=$(_get_demos_from_file ${demo_list_file})
481     _queue_add_demo_jobs ${QUEUE_FILE_DEMOS} ${timeout} ${demos[@]}
482     if [[ ${compress} == "true" ]]; then
483         for v in ${demos[@]}; do
484             videos+=("video/$(basename ${v} | sed 's/.dem$/_000.ogv/')")
485         done
486         _queue_add_compression_jobs ${QUEUE_FILE_COMPRESSING} "mp4" "${videos[@]}"
487     fi
488     _run_demo_jobs ${QUEUE_FILE_DEMOS} "summary" && \
489         _check_if_compress ${compress} "${videos[@]}"
490 }
491
492 process_single() {
493     if [[ ! $1 ]]; then
494         echo "[ ERROR ] Demo name required"
495         exit 1
496     fi
497     local demo_file=$1
498     local timeout=${JOB_TIMEOUT}
499     if [[ $2 =~ ${REGEX_DURATION} ]]; then
500         timeout=$2; shift
501     fi
502     local compress=${COMPRESS}
503     if [[ $2 == "--compress" ]]; then
504         compress="true"
505     fi
506     echo "[ INFO ] Using '${demo_file}' with a timeout of ${timeout}"
507     _queue_add_demo_jobs ${QUEUE_FILE_DEMOS} ${timeout} ${demo_file}
508     if [[ ${compress} == "true" ]]; then
509         local video_file="video/$(basename "${demo_file}" .dem)_000.ogv"
510         _queue_add_compression_jobs ${QUEUE_FILE_COMPRESSING} "mp4" "${video_file}"
511     fi
512     _run_demo_jobs ${QUEUE_FILE_DEMOS} "summary" && \
513         _check_if_compress ${compress} ${video_file}
514 }
515
516 _version() {
517     echo ${VERSION}
518 }
519
520 _help() {
521     echo "./encode-demos.sh
522
523 FLAGS
524
525     --version                                   prints the version string
526
527 COMMANDS
528
529     Encoding
530     --------
531     batch  [demos.txt] [timeout] [--compress]           batch process a list of demos from file relative to \$USERDIR/data
532     single <demo> [timeout] [--compress]                process a single demo file in \$USERDIR/data. ex: demos/cool.dem
533                                                         'timeout' does not include '--compress', compress starts a new job
534     Compression
535     -----------
536     compress <video> [mp4|webm] [--cleanup]             compress an encoded ogv in \$USERDIR/data, ex: video/cool.ogv
537
538     Convert
539     -----------
540     gif <video> [fps] [width] [start] [length] [loop]   convert a video to gif in \$USERDIR/data, ex: video/cool.ogv
541
542     Job Management
543     --------------
544     grep <keyword>                                      grep the server logs of the workers
545     kkill <keyword>                                     keyword kill, kill a worker if string is matched
546     list [-f]                                           list currently active/queued/completed jobs
547     log [-f]                                            tail the current log (-f follows log)
548
549 EXAMPLES
550
551     # outputs \$USERDIR/data/video/2015-06-11_00-26_solarium.ogv (very large)
552     ./encode-demos.sh single demos/2015-06-11_00-26_solarium.dem
553
554     # outputs \$USERDIR/data/video/2015-06-11_00-26_solarium.mp4 (optimal for youtube)
555     ./encode-demos.sh single demos/2015-06-11_00-26_solarium.dem --compress
556
557     # batch
558     ./encode-demos.sh batch demos.txt --compress
559
560     # compress a video in \$USERDIR/data (outputs test.mp4, and deletes the original)
561     ./encode-demos.sh compress video/test.ogv --cleanup
562
563     # convert video to gif (14 fps, 640 width, start at 4s, length of 4s, loop 100 times)
564     ./encode-demos.sh gif video/2017-04-01_11-53_afterslime_000.ogv 14 640 4 4 100
565
566     # list jobs
567     ./encode-demos.sh list
568
569     # inspect worker server logs
570     ./encode-demos.sh grep \"connected\"
571
572     # follow a completed job log
573     ./encode-demos.sh log -f
574
575     # Override the path to Xonotic (assumed from relative location of this script)
576     XONOTIC_DIR=\$HOME/some/other/dir; ./misc/tools/encode-demos.sh --version
577 "
578 }
579
580 # Init
581 ######
582
583 # Allow for overriding the path assumption
584 # XONOTIC_DIR=$HOME/some/other/dir; ./misc/tools/encode-demos.sh --version
585 if [[ -z ${XONOTIC_DIR} ]]; then
586     _get_xonotic_dir
587 else
588     _check_xonotic_dir ${XONOTIC_DIR}
589 fi
590
591 case $1 in
592     # flags
593     '--version')        _version;;
594     ## commands
595     # encoding
596     'batch')            _run_xvfb; process_batch $2 $3 $4;;
597     'single')           _run_xvfb; process_single $2 $3 $4;;
598     # compression
599     'compress')         compress $2 $3 $4;;
600     # convert
601     'gif')              create_gif $2 $3 $4 $5 $6 $7;;
602     # monitoring/management
603     'grep')             log_keyword_grep 'normal' $2;;
604     'kkill')            log_killer_keyword $2;;
605     'list')             list_jobs $2;;
606     'log')              log_completed_jobs $2;;
607     # default
608     *)                  _help; exit 0;;
609 esac