#! /usr/bin/env python3 # encoding: utf-8 import json import os import over import pathlib import re import tempfile import time import version import aux import multiprocessing Command = over.cmd.Command # -------------------------------------------------- X264_BANNED_PIXFMTS = {"bgr24", "yuv422p"} # -------------------------------------------------- # see doc/command_assembler.png command = over.types.ndict() command.identify = Command("ffprobe", "-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", "INFILE") command.normalize_prepass = Command("ffmpeg", "-i", "INFILE", "MAP", "-filter:a", "loudnorm=i=-23.0:tp=-2.0:lra=7.0:print_format=json", "-f", "null", "/dev/null") command.encode_generic = Command("ffmpeg", "FPS", "CUT_FROM", "-i", "INFILE", "-threads", str(multiprocessing.cpu_count()), "CUT_TO", "MAP", "VIDEO", "AUDIO", "-sn", "OUTFILE") command.sub_mp3 = Command("-codec:a", "libmp3lame", "NORMALIZE") command.sub_aac = Command("-codec:a", "aac", "NORMALIZE") command.sub_opus = Command("-codec:a", "libopus", "NORMALIZE") command.sub_vorbis = Command("-codec:a", "libvorbis", "-qscale:a", "QUALITY", "NORMALIZE") command.sub_pcm = Command("-codec:a", "pcm_s16le", "NORMALIZE") command.sub_x264 = Command("PIXFMT", "-codec:v", "libx264", "-preset", "PRESET", "-crf", "QUALITY", "-profile:v", "high", "-level", "4.2", "VFILTER") command.sub_x265 = Command("-codec:v", "libx265", "-preset", "PRESET", "-crf", "QUALITY", "VFILTER") command.sub_vp9 = Command("-codec:v", "libvpx-vp9", "-crf", "QUALITY", "-b:v", "0", "VFILTER") command.sub_av1 = Command("-strict", "experimental", "-codec:v", "libaom-av1", "-crf", "QUALITY", "-b:v", "0", "VFILTER") command.sub_normalize = Command("-filter:a", "LOUDNORM_INCANTATION", "-ar", "48k") command.sub_vfilter = Command("-filter:v", "ARGS") command.merge_chapters = Command("mkvmerge", "--chapters", "CHAPTERS_FILE", "-o", "OUTFILE", "INFILE") command.force_yuv420p = Command("-pix_fmt", "yuv420p") command.sub_copy_audio = Command("-codec:a", "copy") command.sub_copy_video = Command("-codec:v", "copy") # -------------------------------------------------- if __name__ == "__main__": main = over.app.Main("over-video", version.str, "AO-JSL", features={"config_file": True}) main.add_option("audio", "Audio codec to use, either opus<.>, vorbis<.>, aac<.>, pcm<.>, copy<.> or drop<.>.", str, ["opus"], abbr="a", count=1) main.add_option("audio-quality", "Audio encoding quality with -1<.> being the worst and 10<.> being the best. Ignored by --audio<.> opus<.>.", float, [4], abbr="q", count=1) main.add_option("video", "Video codec to use, either av1<.>, vp9<.>, x265<.>, x264<.>, copy<.> or drop<.>.", str, ["vp9"], abbr="v", count=1) main.add_option("video-preset", "Video encoding preset to use by --video<.> x264<.> and x265<.>.", str, ["slow"], abbr="P", count=1) main.add_option("video-quality", "Video encoding quality (CRF). Use 0<.>-51<.> for --video<.> x264<.> and x265<.> (0<.> being lossless, 18<.>-28<.> is reasonable) and 0<.>-63<.> for --video<.> av1<.> or vp9<.> (0<.> being highest, 15<.>-35<.> typical, and 31<.> recommended for HD video).", float, [31], abbr="Q", count=1) main.add_option("container", "The initial container type. Either mkv<.> or webm<.> (or anything else supported by ffmpeg).", str, ["mkv"], count=1) main.add_option("normalize", "Normalize the audio track without clipping. May use dynamic range compression.", bool, [True], abbr="n") main.add_option("ffmpeg-vfilter", 'Raw ffmpeg -filter:v options, e.g. "scale=1280:trunc(ow/a/2)*2,transpose=dir=1<.>"', str, abbr="F", count=1) main.add_option("ffmpeg-map", "Raw ffmpeg -map<.> options, e.g. --map<.> 0:1<.> --map<.> 0:2<.>. This is a drop-in fix until we get proper stream selection.", str, abbr="M", overwrite=False, count=1) main.add_option("chapters", "Path to a Matroska chapters file. See [Chapters<.>].", str, count=1) main.add_option("cut", "Start timestamp and the duration of the portion to use. Uses native ffmpeg -ss<.> and -to<.> format, so it's either seconds from start or [:]:[.<...]<.>. Example: --cut<.> 25 10<.> uses 10 seconds of video starting at 25s, --cut<.> 1:10:45 13:9.5<.> uses video from 4245s to 5034.5s.", over.callback.strings, abbr="X", count=2) main.add_option("fps", "Override input framerate.", float, abbr="f", count=1) main.add_option("output", "Force an output filename. Note that this overrides --container<.> as we're relying on ffmpeg's container detection by reading the suffix. Pass an empty string to use the container's default suffix.", str, [""], count=1) main.add_option("force-5718", "Force bug #5718 workaround for --audio<.> opus<.>", bool) main.add_option("move-source", "Move source file to this directory after conversion. Pass an empty string to disable.", str, ["processed"], count=1) main.add_option("dump-commands", "Print ffmpeg commands that would be executed. If --normalize<.> is in effect, the normalization pre-pass will still be performed so that the proper volume correction can be computed.", bool, [False], abbr="D", in_cfg_file=False) main.add_option("probe", "Print the raw dict (JSON-esque) output of ffprobe and exit.", bool, [False], abbr="p", in_cfg_file=False) main.add_option("armed", "Perform the suggested action.", bool, [False], abbr="A", in_cfg_file=False) main.add_doc("Description", ["A video converter meant to coerce all video formats into one format with properly normalized audio. It can also be used to extract audio from video files, resizing, or very basic cutting."]) main.add_doc("Known good encoder settings", ["vp9<.>: --video<.> vp9<.> --video-quality<.> 31<.> --audio<.> opus<.> (this is the default and should provide best overall results)", "x264<.>: --video<.> x264<.> --video-preset<.> slow<.> --video-quality<.> 22<.>", "x265<.>: --video<.> x265<.> --video-preset<.> medium<.> --video-quality<.> 20<.>"]) main.add_doc("Performance", ["Good bitstreams take obscene amounts of CPU time to produce. See /doc/codec-comparison.tsv for a table of various configs encoding a 1080p video.", "AV1 is currently unusable due to the amount of time it takes to produce a single frame."]) main.add_doc("Chapters", ["Over-video can add chapters to a MKV file. The definition is taken from a separate file with the following syntax:", "CHAPTERxx<.>=HH:MM:SS.SSS\n CHAPTERxx<.>NAME=chapter's name\n (...)", "(where xx<.> is a counter starting from 01)"]) main.setup() # -------------------------------------------------- # cfg checks files = over.types.ndict() audio_words = [] video_words = [] files.container = main.cfg.container if main.cfg.audio in ("copy", "drop"): audio_words.append("%s<.>" %(main.cfg.audio)) else: audio_words.append("codec<.>=%s<.>" %(main.cfg.audio)) if main.cfg.audio == "vorbis": audio_words.append("quality<.>=%.1f<.>" %(main.cfg.audio_quality)) if main.cfg.normalize: audio_words.append("normalize<.>") if main.cfg.video in ("copy", "drop"): video_words.append("%s<.>" %(main.cfg.video)) else: video_words.append("codec<.>=%s<.>" %(main.cfg.video)) video_words.append("quality<.>=%.1f<.>" %(main.cfg.video_quality)) if main.cfg.video_preset and main.cfg.video in ("x264", "x265"): video_words.append("preset<.>=%s<.>" %(main.cfg.video_preset)) if main.cfg.ffmpeg_vfilter: video_words.append("vfilter<.>=%s<.>" %(main.cfg.ffmpeg_vfilter)) if main.cfg.video == "drop": if main.cfg.audio == "pcm": files.container = "wav" elif main.cfg.audio == "vorbis": files.container = "ogg" elif main.cfg.audio == "opus": files.container = "opus" if main.cfg.chapters: if main.cfg.container != "mkv": main.print("unable to use --chapters<.> with --container<.> %s<.>" %(main.cfg.container)) main.exit(1) files.container = "mka" if not os.path.exists(main.cfg.chapters): raise FileNotFoundError(main.cfg.chapters) main.print("settings", main.print.tl.start, end=":\n") main.print("audio: %s" %(", ".join(audio_words))) main.print("video: %s" %(", ".join(video_words))) main.print("container: type<.>=%s<.>" %(files.container)) if main.cfg.move_source: main.print("move source files to %s<.>/" %(main.cfg.move_source)) if main.cfg.audio not in ("drop", "copy", "pcm", "vorbis", "opus", "aac", "mp3"): raise ValueError("unknown audio codec: %s" %(main.cfg.audio)) if main.cfg.video not in ("drop", "copy", "x264", "x265", "vp9", "av1"): raise ValueError("unknown video codec: %s" %(main.cfg.video)) if not main.targets: main.print("no files specified", main.print.tl.warn) for tgt in main.targets: print() files.infile = aux.to_Path(tgt) files.tmpfile = aux.to_Path(tempfile.mktemp(suffix="." + files.container, dir=".")) if main.cfg.output: files.outfile = main.cfg.output else: files.outfile = files.infile.parent / (str(files.infile.stem) + "." + files.container) files.move_infile_to = aux.to_Path(main.cfg.move_source) / files.infile.name if main.cfg.move_source else None if not os.path.exists(tgt) or os.path.isdir(tgt): main.print("target %s<.> is not a readable file<.>, skipping" %(tgt), main.print.tl.fail) continue original_filesize = over.text.Unit(files.infile.stat().st_size, "o") main.print("processing %s<.> (%s)" %(tgt, original_filesize), main.print.tl.start) # -------------------------------------------------- # identify the input file command.identify.reset() command.identify.INFILE = "file:" + str(files.infile) command.identify.run() identify_raw = command.identify.get_all_output().decode("utf-8") identify_dict = json.loads(identify_raw) if main.cfg.probe: print(identify_raw) continue info = over.types.ndict() try: info.duration = float(identify_dict["format"]["duration"]) video_streams = [s for s in identify_dict["streams"] if s["codec_type"] == "video"] audio_streams = [s for s in identify_dict["streams"] if s["codec_type"] == "audio"] amount_vs = len(video_streams) amount_as = len(audio_streams) if amount_vs > 1: main.print('detected %d<.> video streams, picking the "best" one (see man 1 ffmpeg, section STREAM SELECTION)' %(amount_vs), main.print.tl.warn) if amount_as > 1: main.print('detected %d<.> audio streams, picking the "best" one (see man 1 ffmpeg, section STREAM SELECTION)' %(amount_as), main.print.tl.warn) if video_streams: # ffmpeg picks the stream with the highest pixel count and then the lowest index video_streams.sort(key=lambda s: s["width"] * s["height"], reverse=True) video = video_streams[0] info.video_codec = video["codec_name"] info.video_size_x = video["width"] info.video_size_y = video["height"] info.video_fps = over.text.Unit(aux.parse_fps(video["r_frame_rate"]), "Hz") if "bit_rate" in video: info.video_bitrate = over.text.Unit(video["bit_rate"], "b/s") elif "tags" in video and "BPS" in video["tags"]: info.video_bitrate = over.text.Unit(int(video["tags"]["BPS"]), "b/s") else: info.video_bitrate = "??<.>" info.pixel_fmt = video["pix_fmt"] else: info.video_fps = 30 # faked for progress bars if audio_streams: # ffmpeg picks the stream with the most channels and then the lowest index audio_streams.sort(key=lambda s: s["channels"], reverse=True) audio = audio_streams[0] info.audio_codec = audio["codec_name"] info.audio_channels = audio["channels"] info.audio_channel_layout = audio["channel_layout"] if "channel_layout" in audio else "don't care" info.audio_samplerate = over.text.Unit(audio["sample_rate"], "Hz") info.audio_language = audio["tags"]["language"] if "tags" in audio and "language" in audio["tags"] else "und" info.audio_bitrate = over.text.Unit(audio["bit_rate"], "b/s") if "bit_rate" in audio else "??<.>" else: info.audio_channel_layout = None except: main.print("exception while reading identify_dict, dump follows", main.print.tl.fail) print(identify_dict) raise if video_streams: main.print("video<.>: size=%d<.>x%d<.> px, framerate=%s, codec=%s, bitrate=%s" %(info.video_size_x, info.video_size_y, info.video_fps, info.video_codec, info.video_bitrate)) else: main.print("video<.>: None<.>", main.print.tl.warn) if audio_streams: main.print("audio<.>: channels=%d<.>, samplerate=%s, codec=%s, bitrate=%s, language=%s" %(info.audio_channels, info.audio_samplerate, info.audio_codec, info.audio_bitrate, info.audio_language)) else: main.print("audio<.>: None<.>", main.print.tl.warn) # prepare input stream mapping (which needs to be correct even for the normalization pre-pass) if main.cfg.ffmpeg_map: info.map_command = [] for m in main.cfg.ffmpeg_map: info.map_command.append("-map") info.map_command.append(m) else: info.map_command = None # -------------------------------------------------- # normalization pre-pass if audio_streams and main.cfg.normalize and (main.cfg.armed or main.cfg.dump_commands) and (main.cfg.audio not in ["copy", "drop"]): main.print("running normalization pre-pass") command.normalize_prepass.reset() command.normalize_prepass.INFILE = "file:" + str(files.infile) command.normalize_prepass.MAP = info.map_command command.normalize_prepass.run(stderr=True) pb = over.text.ProgressBar( "§%a [§=a>§ A] §sa (Trem=§TA)", { "a": { "unit": "s", "top": info.duration, "precision": 1, "min_width_raw": 0, "min_width_rate": 0, "min_width_time": 0 } } ) pb.render() output_buffer = [] while True: time.sleep(.25) out = command.normalize_prepass.get_output() if out: output_buffer.append(out) if b"time=" in out: t = aux.parse_time(out) pb.set("a", t) pb.render() elif out is None: break pb.end() output = b"".join(output_buffer) # decode the JSON dump from loudnorm if output.count(b"{") == 1 and output.count(b"}") == 1: loudnorm_dict = json.loads(output[output.index(b"{"):].decode("ascii")) info.loudnorm = over.types.ndict({k: aux.float_or_string(v) for k, v in loudnorm_dict.items()}) else: main.print("unexpected ffmpeg output<.>, dump follows", main.print.tl.fail, end=":\n") print(output.decode("utf-8")) raise RuntimeError main.print("detected true peak %.1f dB" %(info.loudnorm.input_tp)) info.normalize_command = command.sub_normalize info.normalize_command.reset() info.normalize_command.LOUDNORM_INCANTATION = "loudnorm=i=-23.0:lra=7.0:tp=-2.0:offset=0.0:measured_i=%.02f:measured_lra=%.02f:measured_tp=%.02f:measured_thresh=%.02f:linear=true" %( info.loudnorm.input_i, info.loudnorm.input_lra, info.loudnorm.input_tp, info.loudnorm.input_thresh ) else: info.normalize_command = None # -------------------------------------------------- # main command assembly encode_cmd = command.encode_generic encode_cmd.reset() encode_cmd.INFILE = "file:" + str(files.infile) encode_cmd.OUTFILE = files.tmpfile encode_cmd.FPS = ["-r", main.cfg.fps] if main.cfg.fps else None encode_cmd.CUT_FROM = ["-ss", main.cfg.cut[0]] if main.cfg.cut else None encode_cmd.CUT_TO = ["-to", main.cfg.cut[1]] if main.cfg.cut else None if main.cfg.audio == "copy": encode_cmd.AUDIO = command.sub_copy_audio elif main.cfg.audio == "drop": encode_cmd.AUDIO = "-an" elif main.cfg.audio == "pcm": command.sub_pcm.reset() command.sub_pcm.NORMALIZE = info.normalize_command encode_cmd.AUDIO = command.sub_pcm elif main.cfg.audio == "vorbis": command.sub_vorbis.reset() command.sub_vorbis.QUALITY = main.cfg.audio_quality command.sub_vorbis.NORMALIZE = info.normalize_command encode_cmd.AUDIO = command.sub_vorbis elif main.cfg.audio == "aac": command.sub_aac.reset() command.sub_aac.NORMALIZE = info.normalize_command encode_cmd.AUDIO = command.sub_aac elif main.cfg.audio == "opus": command.sub_opus.reset() command.sub_opus.NORMALIZE = info.normalize_command # workaround of https://trac.ffmpeg.org/ticket/5718 if info.audio_channel_layout == "5.1(side)" or main.cfg.force_5718: main.print("applying #5718 workaround<.>", main.print.tl.warn) command.sub_opus.sequence.append("-filter:a") command.sub_opus.sequence.append("channelmap=channel_layout=5.1") encode_cmd.AUDIO = command.sub_opus elif main.cfg.audio == "mp3": command.sub_mp3.reset() command.sub_mp3.NORMALIZE = info.normalize_command encode_cmd.AUDIO = command.sub_mp3 if main.cfg.ffmpeg_vfilter: info.vfilter_command = command.sub_vfilter info.vfilter_command.reset() info.vfilter_command.ARGS = main.cfg.ffmpeg_vfilter else: info.vfilter_command = None encode_cmd.MAP = info.map_command if main.cfg.video == "copy": encode_cmd.VIDEO = command.sub_copy_video elif main.cfg.video == "drop": encode_cmd.VIDEO = "-vn" elif main.cfg.video == "vp9": command.sub_vp9.reset() command.sub_vp9.QUALITY = main.cfg.video_quality command.sub_vp9.VFILTER = info.vfilter_command encode_cmd.VIDEO = command.sub_vp9 elif main.cfg.video == "av1": command.sub_av1.reset() command.sub_av1.QUALITY = main.cfg.video_quality command.sub_av1.VFILTER = info.vfilter_command encode_cmd.VIDEO = command.sub_av1 elif main.cfg.video == "x264": command.sub_x264.reset() command.sub_x264.QUALITY = main.cfg.video_quality command.sub_x264.PRESET = main.cfg.video_preset command.sub_x264.VFILTER = info.vfilter_command if info.pixel_fmt in X264_BANNED_PIXFMTS: main.print("source pixel format %s<.> is incompatible with x264, forcing yuv420p<.>" %(info.pixel_fmt), main.print.tl.warn) command.sub_x264.PIXFMT = command.force_yuv420p else: command.sub_x264.PIXFMT = None encode_cmd.VIDEO = command.sub_x264 elif main.cfg.video == "x265": command.sub_x265.reset() command.sub_x265.QUALITY = main.cfg.video_quality command.sub_x265.PRESET = main.cfg.video_preset command.sub_x265.VFILTER = info.vfilter_command encode_cmd.VIDEO = command.sub_x265 # -------------------------------------------------- # run the command iff armed if main.cfg.dump_commands or main.cfg.armed: cmd = " ".join(encode_cmd.dump(pretty=True)) if main.cfg.armed: main.print("executing %s<.>" %(cmd), main.print.tl.start) else: main.print("will execute %s<.>" %(cmd)) else: main.print("will encode into %s<.>" %(files.tmpfile)) if main.cfg.armed: pb = over.text.ProgressBar( "§%a §rs [§=a>§ A] §sa (§ss) (Sest=§zs, Trem=§TA)", { "a": { "unit": "s", "top": int(info.duration), "precision": 1, "min_width_rate": 9 }, "s": { "unit": "o", # octets are cool "top": None, # size is unknown at the start but will be estimated during updates "precision": 1, "min_width_raw": 9, "min_width_rate": 11 } } ) encode_cmd.run(stderr=True) while True: time.sleep(.25) out = encode_cmd.get_output() if out: if b"time=" in out: t = aux.parse_time(out) try: pb.set("a", int(t)) except ZeroDivisionError: print(out) elif out is None: break try: pb.set("s", files.tmpfile.stat().st_size) except FileNotFoundError: # a race condition with ffmpeg pass pb.render() new_filesize = over.text.Unit(files.tmpfile.stat().st_size, "o") pb.end() if encode_cmd.returncode == 0: main.print("encoding finished: %s -> %s" %(original_filesize, new_filesize), main.print.tl.done) else: main.print("encoding failed<.>, ffmpeg returned %d<.>" %(encode_cmd.returncode), main.print.tl.fail) raise RuntimeError # -------------------------------------------------- # inject chapters if main.cfg.chapters: if main.cfg.dump_commands or main.cfg.armed: files.tmpfile_nochapters = aux.to_Path("nochapter-" + str(files.tmpfile)) # really pathlib? command.merge_chapters.reset() command.merge_chapters.CHAPTERS_FILE = main.cfg.chapters command.merge_chapters.OUTFILE = files.tmpfile command.merge_chapters.INFILE = files.tmpfile_nochapters cmd = " ".join(command.merge_chapters.dump(pretty=True)) if main.cfg.armed: main.print("moving %s<.> -> %s<.>" %(files.tmpfile, files.tmpfile_nochapters), main.print.tl.start) files.tmpfile.rename(files.tmpfile_nochapters) main.print("executing %s<.>" %(cmd), main.print.tl.start) command.merge_chapters.run() _ = command.merge_chapters.get_all_output() main.print("deleting %s<.>" %(files.tmpfile_nochapters)) files.tmpfile_nochapters.unlink() else: main.print("will move %s<.> -> %s<.>" %(files.tmpfile, files.tmpfile_nochapters)) main.print("will execute %s<.>" %(cmd)) main.print("will delete %s<.>" %(files.tmpfile_nochapters)) else: main.print("will add chapters from %s<.>" %(main.cfg.chapters)) # -------------------------------------------------- # shuffle files around if main.cfg.move_source: move_to_dir = pathlib.Path(main.cfg.move_source) if not move_to_dir.is_dir(): if main.cfg.armed: main.print("creating directory %s<.>" %(move_to_dir), main.print.tl.start) move_to_dir.mkdir() else: main.print("will create directory %s<.>" %(move_to_dir)) if files.move_infile_to: if main.cfg.armed: main.print("moving %s<.> -> %s<.>" %(files.infile, files.move_infile_to), main.print.tl.start) files.infile.rename(files.move_infile_to) else: main.print("will move %s<.> -> %s<.>" %(files.infile, files.move_infile_to)) if main.cfg.armed: main.print("moving %s<.> -> %s<.>" %(files.tmpfile, files.outfile), main.print.tl.start) files.tmpfile.rename(files.outfile) else: main.print("will move %s<.> -> %s<.>" %(files.tmpfile, files.outfile))