diff --git a/over-video.py b/over-video.py index 4609552..f2887d1 100755 --- a/over-video.py +++ b/over-video.py @@ -1,141 +1,693 @@ #! /usr/bin/env python3 # encoding: utf-8 +import aux +from dataclasses import dataclass, field +import enum import json +import multiprocessing # for cpu_count import os import over import pathlib import re import tempfile import time +import typing import version -import aux -import multiprocessing Command = over.cmd.Command -# -------------------------------------------------- -X264_BANNED_PIXFMTS = {"bgr24", "yuv422p"} +## +## Dynamic command templates: functions and classes +## -# -------------------------------------------------- +@dataclass +class CodecPar: + name: str + prefix: str + mandatory: bool + default: typing.Any + is_valid: typing.Callable + doc: str + + def resolve(self, raw): + """ + Takes an argument from the user and resolves it into a command line parameter + and value. + + E.g. "45" -> ["-crf", "45"] + + Mandatory CodecPars will raise a ValueError if raw is None. Non-mandatory + CodecPars will substitude their default if None, and return an empty list + if neither mandatory nor have a default. + """ + + if raw is None: + value = self.default + else: + value = raw + + if value is None: + if self.mandatory: + raise ValueError("%s must be set" %(self.name)) + else: + return [] + + if not self.is_valid(value): + raise ValueError("%s is not a valid %s" %(value, self.name)) + + return [self.prefix, value] + +class Codec: + def __init__(self, name: str, doc: str, prelude: list, parameters: list): + """ + Represents a portion of an ffmpeg incantation that defines a + particular codec and its parameters. + + >>> c = Codec( + "x265", + ["-c:v", "libx265"], + [ + CodecPar("CRF", "-crf", True, None, StrIntInRange(0, 51), "Constant Rate Factor: 0 is lossless, 18-25 generally very good."), + CodecPar("SPEED", "-preset", True, "slow", IsEnum(x264_presets), "Encoder preset: one of " + ", ".join(x264_presets)), + ... + ] + ) + >>> c.parse("x264:24,medium") # incorrect name, returns nothing + >>> c.parse("x265:20,superslow") + ["-c:v", "libx265", "-crf", "20", "-preset", "superslow"] + + :param name: Codec's name visible to the user + :param prelude: List of shell words that prefix the codec + :param parameters: List of CodecPar instances + """ + + self.name = name + self.doc = doc + self.prelude = prelude + self.parameters = parameters + + def parse(self, line): + """ + Parses the config line and produces a list of shell words. + + If the codec name does not match this Codec, returns None. + """ + + name, *rest = line.split(":") + args = rest[0].split(",") if rest else [] + + if name != self.name: + return None + + cmd = self.prelude[:] + + for i, parameter in enumerate(self.parameters): + if i < len(args): + arg = args[i] + else: + arg = None + + part = parameter.resolve(arg) + cmd.extend(part) + + return cmd + +codec_par_colors = ["R", "r", "Y", "y", "C", "c"] + +@dataclass +class CodecRouter: + """ + Holds a list of known codecs and their parameters. Takes a codec definition + line and returns (the codec's name, appropriate ffmpeg command part). + """ + + color: str + codecs: list = field(default_factory=list) + + def add_codec(self, codec): + self.codecs.append(codec) + + def get_command(self, config): + for codec in self.codecs: + cmd = codec.parse(config) + + if cmd: + return (codec.name, cmd) + + def help(self, indent=0): + """ + Returns a help text for all codecs. + """ + + lines = [] + + for codec in self.codecs: + lines.append("‣ <%s>%s<.>" %(self.color, codec.name)) + lines.append(" %s" %(codec.doc)) + + usage = [] + par_lines = [] + + for i, par in enumerate(codec.parameters): + usage.append("<%s>%s<.>" %(codec_par_colors[i], par.name)) + par_line_parts = [" "] + + par_line_parts.append("<%s>%s<.>: %s" %(codec_par_colors[i], par.name, par.doc)) + + if par.default is not None: + par_line_parts.append(" (default %s)<.>" %(par.default)) + + if not par.mandatory: + par_line_parts.append(" (optional)<.>") + + par_lines.append("".join(par_line_parts)) + + if usage: + lines.append(" Incantation: <%s>%s<.>:%s" %(self.color, codec.name, ",".join(usage))) + else: + lines.append(" Incantation: <%s>%s<.>" %(self.color, codec.name)) + + lines.extend(par_lines) + lines.append("") + + return "\n".join(" "*indent + line for line in lines) + +# Validators + +@dataclass +class StrIntInRange: + lowest: int + highest: int + + def __call__(self, value): + try: + value = int(value) + except: + return False + + return value >= self.lowest and value <= self.highest + +@dataclass +class StrFloatInRange: + lowest: float + highest: float + + def __call__(self, value): + try: + value = float(value) + except: + return False + + return value >= self.lowest and value <= self.highest + +@dataclass +class IsEnum: + items: list + + def __call__(self, value): + return value in self.items + +def IntK(value): + if value[-1].lower() == "k": + value = value[:-1] + + try: + _ = int(value) + + return True + + except: + return False + +## +## Dynamic command templates: codec definitions +## + +x264_presets = [ + "ultrafast", + "superfast", + "veryfast", + "faster", + "fast", + "medium", + "slow", + "slower", + "veryslow", + "placebo" +] + +x264_tunes = [ + "film", + "animation", + "grain", + "stillimage", + "fastdecode", + "zerolatency", + "psnr", + "ssim" +] + +x265_tunes = [ + "animation", + "grain", + "fastdecode", + "zerolatency", + "psnr", + "ssim" +] + +codec_x265 = Codec( + "x265", + "HEVC encoded using x265. Very good quality and decent encode times.", + ["-codec:v", "libx265"], + [ + CodecPar( + "CRF", + "-crf", + True, + None, + StrIntInRange(0, 51), + "Constant Rate Factor: 0 is lossless, 18-25 generally very good, 51 worst." + ), + CodecPar( + "SPEED", + "-preset", + True, + "slow", + IsEnum(x264_presets), + "Encoder preset, one of " + ", ".join(x264_presets) + "." + ), + CodecPar( + "TUNE", + "-tune", + False, + None, + IsEnum(x265_tunes), + "Optional tune hint, one of "+ ", ".join(x265_tunes) + "." + ) + ] +) + +codec_x264 = Codec( + "x264", + "AVC encoded using x264. Decent quality, fast encodes and very portable.", + ["-codec:v", "libx264"], + [ + CodecPar( + "CRF", + "-crf", + True, + None, + StrIntInRange(0, 63), + "Constant Rate Factor: 0 is lossless, 20-28 generally very good, 63 worst." + ), + CodecPar( + "SPEED", + "-preset", + True, + "slow", + IsEnum(x264_presets), + "Encoder preset, one of " + ", ".join(x264_presets) + "." + ), + CodecPar( + "TUNE", + "-tune", + False, + None, + IsEnum(x264_tunes), + "Optional tune hint, one of "+ ", ".join(x264_tunes) + "." + ) + ] +) + +codec_rav1e = Codec( + "rav1e", + "AV1 encoded using rav1e. Excellent quality, glacially slow encode times bordering on unusable.", + ["-codec:v", "librav1e"], + [ + CodecPar( + "QP", + "-qp", + True, + None, + StrIntInRange(0, 255), + "Quantizer: 0 is of highest quality, 80 very good, 100 decent, 255 the worst." + ), + CodecPar( + "SPEED", + "-speed", + True, + None, + StrIntInRange(0, 10), + "Speed (at the expense of quality): 0 is the slowest, 10 fastest." + ), + CodecPar( + "COLS", + "-tile-columns", + True, + "2", + StrIntInRange(1, 10), + "How many columns to tile the video into. More columns give faster encodes (and decodes) at slightly worse quality." + ), + CodecPar( + "ROWS", + "-tile-rows", + True, + "2", + StrIntInRange(1, 10), + "How many rows to tile the video into. More rows give faster encodes (and decodes) at slightly worse quality." + ) + ] +) + +codec_opus = Codec( + "opus", + "Opus audio via libopus VBR. Excellent overall.", + ["-codec:a", "libopus", "-vbr", "on"], + [ + CodecPar( + "ABR", + "-b:a", + True, + "96k", + IntK, + "Average bitrate in bps. The default gives very good results." + ) + ] +) + +codec_vorbis = Codec( + "vorbis", + "Vorbis VBR audio via libvorbis.", + ["-codec:a", "libvorbis"], + [ + CodecPar( + "QUALITY", + "-q:a", + True, + "3.0", + StrFloatInRange(-1, 10), + "Target quality where -1 is the worst and 10 the best. The default gives very good results." + ) + ] +) + +codec_pcm = Codec( + "pcm", + "Uncompressed PCM s16le audio.", + ["-codec:a", "pcm_s16le"], + [] +) + +codec_vcopy = Codec( + "copy", + "Directly copies the source video stream without encoding.", + ["-codec:v", "copy"], + [] +) + +codec_vdrop = Codec( + "drop", + "Drops the source video stream.", + ["-vn"], + [] +) + +codec_acopy = Codec( + "copy", + "Directly copies the source audio stream without encoding.", + ["-codec:a", "copy"], + [] +) + +codec_adrop = Codec( + "drop", + "Drops the source audio stream.", + ["-an"], + [] +) + +codec_scopy = Codec( + "copy", + "Copies over subtitles embedded in the source.", + ["-codec:s", "copy"], + [] +) + +codec_sdrop = Codec( + "drop", + "Drops all source subtitle tracks.", + ["-sn"], + [] +) + +video_codec_router = CodecRouter("M") +video_codec_router.add_codec(codec_x265) +video_codec_router.add_codec(codec_x264) +video_codec_router.add_codec(codec_rav1e) +video_codec_router.add_codec(codec_vcopy) +video_codec_router.add_codec(codec_vdrop) + +audio_codec_router = CodecRouter("C") +audio_codec_router.add_codec(codec_opus) +audio_codec_router.add_codec(codec_vorbis) +audio_codec_router.add_codec(codec_pcm) +audio_codec_router.add_codec(codec_acopy) +audio_codec_router.add_codec(codec_adrop) + +subtitle_codec_router = CodecRouter("Y") +subtitle_codec_router.add_codec(codec_scopy) +subtitle_codec_router.add_codec(codec_sdrop) + +## +## Static command templates +## # see doc/command_assembler.png command = over.types.ndict() command.identify = Command("ffprobe", "-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", "INFILE") command.normalize_prepass = Command("ffmpeg", "-i", "INFILE", "MAP", "-filter:a", "loudnorm=i=-23.0:tp=-2.0:lra=7.0:print_format=json", "-f", "null", "/dev/null") -command.encode_generic = Command("ffmpeg", "FPS", "CUT_FROM", "-i", "INFILE", "-threads", str(multiprocessing.cpu_count()), "CUT_TO", "MAP", "VIDEO", "AUDIO", "-sn", "OUTFILE") -command.sub_mp3 = Command("-codec:a", "libmp3lame", "NORMALIZE") -command.sub_aac = Command("-codec:a", "aac", "NORMALIZE") -command.sub_opus = Command("-codec:a", "libopus", "-b:a", "96000", "NORMALIZE") -command.sub_vorbis = Command("-codec:a", "libvorbis", "-qscale:a", "QUALITY", "NORMALIZE") -command.sub_pcm = Command("-codec:a", "pcm_s16le", "NORMALIZE") -command.sub_x264 = Command("PIXFMT", "-codec:v", "libx264", "-preset", "PRESET", "-crf", "QUALITY", "-profile:v", "high", "-level", "4.2", "VFILTER") -command.sub_x265 = Command("-codec:v", "libx265", "-preset", "PRESET", "-crf", "QUALITY", "VFILTER") -command.sub_vp9 = Command("-codec:v", "libvpx-vp9", "-crf", "QUALITY", "-b:v", "0", "VFILTER") -command.sub_av1 = Command("-codec:v", "librav1e", "-qp", "QUALITY", "-speed", "4", "-tile-columns", "2", "-tile-rows", "2", "VFILTER") -command.sub_normalize = Command("-filter:a", "LOUDNORM_INCANTATION", "-ar", "48k") +command.encode_generic = Command("ffmpeg", "FPS", "CUT_FROM", "-i", "INFILE", "-threads", str(multiprocessing.cpu_count()), "CUT_TO", "MAP", "VIDEO", "AUDIO", "SUBTITLE", "OUTFILE") command.sub_vfilter = Command("-filter:v", "ARGS") +command.sub_afilter = Command("-filter:a", "ARGS", "-ar", "48k") command.merge_chapters = Command("mkvmerge", "--chapters", "CHAPTERS_FILE", "-o", "OUTFILE", "INFILE") command.force_yuv420p = Command("-pix_fmt", "yuv420p") -command.sub_copy_audio = Command("-codec:a", "copy") -command.sub_copy_video = Command("-codec:v", "copy") + +## +## UI helpers +## + +@dataclass +class ListCodecs: + video_codecs: CodecRouter + audio_codecs: CodecRouter + subtitle_codecs: CodecRouter + + def __call__(self, main): + print("Available codecs and their configuration:\n") + + print(over.text.render("--video<.> codecs:")) + print(over.text.render(self.video_codecs.help(indent=2))) + + print(over.text.render("--audio<.> codecs:")) + print(over.text.render(self.audio_codecs.help(indent=2))) + + print(over.text.render("--subtitle<.> codecs:")) + print(over.text.render(self.subtitle_codecs.help(indent=2))) + + main.exit(True) + +list_codecs = ListCodecs(video_codec_router, audio_codec_router, subtitle_codec_router) + +## +## Functions +## + +def get_preset(presets, preset): + out = [] + + if not presets: + return + + for atom in presets.split(): + if not atom[0] == "@": + raise ValueError("invalid preset: " + atom) + + name, data = atom[1:].split(":", 1) + + if name != preset: + continue + + for statement in data.split(";"): + key, value = statement.split("=", 1) + out.append((key, value)) + + return out + +# -------------------------------------------------- + +factory_presets = " ".join([ + "@share:video=x264:23,slow;audio=opus;container=mp4;ffmpeg-vfilter=scale=1280:trunc(ow/a/2)*2", + "@share-portrait:video=x264:23,slow;audio=opus;container=mp4;ffmpeg-vfilter=scale=720:trunc(ow/a/2)*2", + "@archive-hq:video=x265:23,slow;audio=opus;container=mkv", + "@archive:video=x265:25,medium;audio=opus;container=mkv", + "@archive-animation:video=x265:27,slow;audio=opus;container=mkv" +]) # -------------------------------------------------- if __name__ == "__main__": - main = over.app.Main("over-video", version.str, "AO-JSL", features={"config_file": True}) - main.add_option("audio", "Audio codec to use, either opus<.>, vorbis<.>, aac<.>, pcm<.>, copy<.> or drop<.>.", str, ["opus"], abbr="a", count=1) - main.add_option("audio-quality", "Audio encoding quality with -1<.> being the worst and 10<.> being the best. Ignored by --audio<.> opus<.>.", float, [4], abbr="q", count=1) - main.add_option("video", "Video codec to use, either av1<.>, vp9<.>, x265<.>, x264<.>, copy<.> or drop<.>.", str, ["av1"], abbr="v", count=1) - main.add_option("video-preset", "Video encoding preset to use by --video<.> x264<.> and x265<.>.", str, ["slow"], abbr="P", count=1) - main.add_option("video-quality", "Video encoding quality. Interpreted as CRF: use 0<.>-51<.> for --video<.> x264<.> and x265<.> (0<.> being lossless, 18<.>-28<.> is reasonable) and 0<.>-63<.> for --video<.> vp9<.> (0<.> being highest, 15<.>-35<.> typical, and 31<.> recommended for HD video). Interpreted as quantizer: use 0<.>-255<.> for --video<.> av1<.> (80<.> being high quality, 100<.> reasonable, 255<.> lowest bitrate).", float, [80], abbr="Q", count=1) - main.add_option("container", "The initial container type. Either mkv<.> or webm<.> (or anything else supported by ffmpeg).", str, ["mkv"], count=1) - main.add_option("normalize", "Normalize the audio track without clipping. May use dynamic range compression.", bool, [True], abbr="n") + if not (over.version.major >= 3): + raise RuntimeError("dev-python/over-3.0 or newer is required") + + main = over.app.Main("over-video", version.str, "GPL-3", features={"config_file": True}) + main.log.format = "<@>: " + main.add_option("presets", "Space-separated list of presets. See the [Presets<.>] chapter for details.", str, [factory_presets], abbr="p", count=1) + main.add_option("preset", "Name of the selected preset.", str, abbr="P", count=1) + main.add_option("codecs", "List all available codecs and their usage.", lambda: list_codecs(main)) + main.add_option("audio", "Audio codec incantation to use. See --codecs<.> for a list.", str, ["opus"], abbr="a", count=1) + main.add_option("video", "Video codec incantation to use. See --codecs<.> for a list.", str, abbr="v", count=1) + main.add_option("subtitle", "Subtitle codec incantation to use. See --codecs<.> for a list.", str, ["drop"], abbr="s", count=1) + main.add_option("container", "Override the target container, e.g. mkv<.>, mp4<.>, or anything else supported by ffmpeg. Over-video uses mkv by default but changes to an appropriate audio container if no video tracks will be in the final output.", str, abbr="c", count=1) + main.add_option("normalize", "Normalize the audio track without clipping. May use dynamic range compression.", bool, [True], abbr="N") main.add_option("ffmpeg-vfilter", 'Raw ffmpeg -filter:v options, e.g. "scale=1280:trunc(ow/a/2)*2,transpose=dir=1<.>"', str, abbr="F", count=1) - main.add_option("ffmpeg-map", "Raw ffmpeg -map<.> options, e.g. --map<.> 0:1<.> --map<.> 0:2<.>. This is a drop-in fix until we get proper stream selection.", str, abbr="M", overwrite=False, count=1) + main.add_option("ffmpeg-afilter", 'Raw ffmpeg -filter:a options, e.g. "pan=mono|c0=FL<.>"', str, abbr="f", count=1) + main.add_option("ffmpeg-map", "Raw ffmpeg -map<.> options, e.g. --map<.> 0:1<.> --map<.> 0:2<.>. This is a drop-in fix until we get proper stream selection.", str, abbr="m", overwrite=False, count=1) main.add_option("chapters", "Path to a Matroska chapters file. See [Chapters<.>].", str, count=1) - main.add_option("cut", "Start timestamp and the duration of the portion to use. Uses native ffmpeg -ss<.> and -to<.> format, so it's either seconds from start or [:]:[.<...]<.>. Example: --cut<.> 25 10<.> uses 10 seconds of video starting at 25s, --cut<.> 1:10:45 13:9.5<.> uses video from 4245s to 5034.5s.", over.callback.strings, abbr="X", count=2) - main.add_option("fps", "Override input framerate.", float, abbr="f", count=1) - main.add_option("output", "Force an output filename. Note that this overrides --container<.> as we're relying on ffmpeg's container detection by reading the suffix. Pass an empty string to use the container's default suffix.", str, [""], count=1) - main.add_option("force-5718", "Force bug #5718 workaround for --audio<.> opus<.>", bool) - main.add_option("move-source", "Move source file to this directory after conversion. Pass an empty string to disable.", str, ["processed"], count=1) - main.add_option("dump-commands", "Print ffmpeg commands that would be executed. If --normalize<.> is in effect, the normalization pre-pass will still be performed so that the proper volume correction can be computed.", bool, [False], abbr="D", in_cfg_file=False) - main.add_option("probe", "Print the raw dict (JSON-esque) output of ffprobe and exit.", bool, [False], abbr="p", in_cfg_file=False) + main.add_option("cut", "Start timestamp and the duration of the portion to use. Uses native ffmpeg -ss<.> and -to<.> format, so it's either seconds from start or [:]:[.<...]<.>. Example: --cut<.> 25 10<.> uses 10 seconds of video starting at 25s, --cut<.> 1:10:45 13:9.5<.> uses video from 4245s to 5034.5s.", over.callback.strings, abbr="x", count=2) + main.add_option("fps", "Override input framerate.", float, count=1) + main.add_option("output", "Force an output filename. Note that this overrides --container<.> as we're relying on ffmpeg's container detection by reading the suffix.", str, [""], abbr="O", count=1) + main.add_option("force-5718", "Force bug #5718 workaround for --audio<.> opus<.>.", bool) + main.add_option("move", "Move source file to this directory after conversion. Pass an empty string to disable.", str, ["processed"], abbr="M", count=1) + main.add_option("dump", "Print ffmpeg commands that would be executed. If --normalize<.> is in effect, the normalization pre-pass will still be performed so that the proper volume correction can be computed.", bool, [False], abbr="D", in_cfg_file=False) + main.add_option("identify", "Print the raw JSON output of ffprobe and exit.", bool, [False], abbr="I", in_cfg_file=False) main.add_option("armed", "Perform the suggested action.", bool, [False], abbr="A", in_cfg_file=False) - main.add_doc("Description", ["A video converter meant to coerce all video formats into one format with properly normalized audio. It can also be used to extract audio from video files, resizing, or very basic cutting."]) - main.add_doc("Known good encoder settings", ["vp9<.>: --video<.> vp9<.> --video-quality<.> 31<.> --audio<.> opus<.> (this is the default and should provide best overall results)", "x264<.>: --video<.> x264<.> --video-preset<.> slow<.> --video-quality<.> 22<.>", "x265<.>: --video<.> x265<.> --video-preset<.> medium<.> --video-quality<.> 20<.>"]) + main.add_doc("Description", ["A video converter meant to coerce all audio-video recordings into one format with consistent audio volume. It can also be used to extract audio from video files, resize video, or for very basic cutting."]) main.add_doc("Performance", ["Good bitstreams take obscene amounts of CPU time to produce. See /doc/codec-comparison.tsv for a table of various configs encoding a 1080p video.", "AV1 is currently unusable due to the amount of time it takes to produce a single frame."]) - main.add_doc("Chapters", ["Over-video can add chapters to a MKV file. The definition is taken from a separate file with the following syntax:", "CHAPTERxx<.>=HH:MM:SS.SSS\n CHAPTERxx<.>NAME=chapter's name\n (...)", "(where xx<.> is a counter starting from 01)"]) + main.add_doc("Chapters", ["Over-video can add chapters to an MKV file. The definition is taken from a separate file with the following syntax:", "CHAPTERxx<.>=HH:MM:SS.SSS\n CHAPTERxx<.>NAME=chapter's name\n (...)", "(where xx<.> is a counter starting from 01)"]) + main.add_doc("Presets", ["The various settings can be saved into a named preset for repeated use. The presets<.> key in the config file holds a space-separated list of preset atoms: @name<.>:option<.>=value<.>;option<.>=value<.>.", "For example, with --presets<.> = \"@anime<.>:video<.>=x265:22,slow,animation<.>;audio<.>=opus<.>;container<.>=mkv<.>\" stored in the config file, a rather verbose invocation with --video<.> ...<.>, --audio<.> ...<.>, and --container<.> ...<.> on the command line can be replaced with just --preset<.> anime<.>. You can specify options on the command line to override parts of a preset."]) main.setup() + # -------------------------------------------------- + # apply presets + options_from_preset = [] + options_from_preset_overriden = [] + + if main.cfg.preset: + preset = get_preset(main.cfg.presets, main.cfg.preset) + + if not preset: + main.log.fail("--preset<.> %s<.> is unknown", main.cfg.preset) + main.exit(1) + + for option_name, value in preset: + option = main.cfg.options[option_name] + + if option.source == over.app.Option_sources.command_line: + options_from_preset_overriden.append(option_name) + else: + options_from_preset.append(option_name) + option.set_value([value], over.app.Option_sources.preset) + # -------------------------------------------------- # cfg checks files = over.types.ndict() audio_words = [] video_words = [] - files.container = main.cfg.container + files.container = "mkv" - if main.cfg.audio in ("copy", "drop"): + if not main.cfg.audio: + main.log.fail("--audio<.> is not set") + main.exit(1) + elif main.cfg.audio in ("copy", "drop"): audio_words.append("%s<.>" %(main.cfg.audio)) else: audio_words.append("codec<.>=%s<.>" %(main.cfg.audio)) - if main.cfg.audio == "vorbis": - audio_words.append("quality<.>=%.1f<.>" %(main.cfg.audio_quality)) + if main.cfg.ffmpeg_afilter: + audio_words.append("afilter<.>=%s<.>" %(main.cfg.ffmpeg_afilter)) if main.cfg.normalize: audio_words.append("normalize<.>") - if main.cfg.video in ("copy", "drop"): + if not main.cfg.video: + main.log.fail("--video<.> is not set") + main.exit(1) + elif main.cfg.video in ("copy", "drop"): video_words.append("%s<.>" %(main.cfg.video)) - else: video_words.append("codec<.>=%s<.>" %(main.cfg.video)) - video_words.append("quality<.>=%.1f<.>" %(main.cfg.video_quality)) - - if main.cfg.video_preset and main.cfg.video in ("x264", "x265"): - video_words.append("preset<.>=%s<.>" %(main.cfg.video_preset)) if main.cfg.ffmpeg_vfilter: video_words.append("vfilter<.>=%s<.>" %(main.cfg.ffmpeg_vfilter)) if main.cfg.video == "drop": - if main.cfg.audio == "pcm": + if main.cfg.audio.startswith("pcm"): files.container = "wav" - elif main.cfg.audio == "vorbis": + elif main.cfg.audio.startswith("vorbis"): files.container = "ogg" - elif main.cfg.audio == "opus": + elif main.cfg.audio.startswith("opus"): files.container = "opus" + if main.cfg.container: + files.container = main.cfg.container + if main.cfg.chapters: - if main.cfg.container != "mkv": - main.print("unable to use --chapters<.> with --container<.> %s<.>" %(main.cfg.container)) + if files.container != "mkv": + main.log.fail("unable to use --chapters<.> with %s<.> containers", files.container) main.exit(1) - files.container = "mka" - if not os.path.exists(main.cfg.chapters): raise FileNotFoundError(main.cfg.chapters) - main.print("settings", main.print.tl.start, end=":\n") - main.print("audio: %s" %(", ".join(audio_words))) - main.print("video: %s" %(", ".join(video_words))) - main.print("container: type<.>=%s<.>" %(files.container)) + if options_from_preset or options_from_preset_overriden: + if options_from_preset: + note_preset = "%s<.>" %("<.>, ".join(options_from_preset)) + else: + note_preset = "nothing<.>" + + if options_from_preset_overriden: + note = "%s loaded from preset %s<.>; %s<.> was overridden by command line" %( + note_preset, + main.cfg.preset, + "<.>, ".join(options_from_preset_overriden) + ) + + else: + note = "%s loaded from preset %s<.>" %( + note_preset, + main.cfg.preset + ) + + main.log.info("settings (%s)", note, end=":\n") + else: + main.log.info("settings", end=":\n") - if main.cfg.move_source: - main.print("move source files to %s<.>/" %(main.cfg.move_source)) + main.log.info("audio: %s", ", ".join(audio_words)) + main.log.info("video: %s", ", ".join(video_words)) + main.log.info("container: %s<.>", files.container) - if main.cfg.audio not in ("drop", "copy", "pcm", "vorbis", "opus", "aac", "mp3"): - raise ValueError("unknown audio codec: %s" %(main.cfg.audio)) + if main.cfg.move: + main.log.info("move source files to %s<.>/", main.cfg.move) - if main.cfg.video not in ("drop", "copy", "x264", "x265", "vp9", "av1"): - raise ValueError("unknown video codec: %s" %(main.cfg.video)) + if not audio_codec_router.get_command(main.cfg.audio): + main.log.fail("unknown --audio<.> codec: %s<.>", main.cfg.audio) + main.exit(1) + + if not video_codec_router.get_command(main.cfg.video): + main.log.fail("unknown --video<.> codec: %s<.>", main.cfg.video) + main.exit(1) + + if not subtitle_codec_router.get_command(main.cfg.subtitle): + main.log.fail("unknown --subtitle<.> codec: %s<.>", main.cfg.subtitle) + main.exit(1) if not main.targets: - main.print("no files specified", main.print.tl.warn) + main.log.warn("no files specified, nothing to do") for tgt in main.targets: print() @@ -146,14 +698,13 @@ if __name__ == "__main__": files.outfile = main.cfg.output else: files.outfile = files.infile.parent / (str(files.infile.stem) + "." + files.container) - files.move_infile_to = aux.to_Path(main.cfg.move_source) / files.infile.name if main.cfg.move_source else None + files.move_infile_to = aux.to_Path(main.cfg.move) / files.infile.name if main.cfg.move else None - if not os.path.exists(tgt) or os.path.isdir(tgt): - main.print("target %s<.> is not a readable file<.>, skipping" %(tgt), main.print.tl.fail) + if (not os.path.exists(tgt)) or os.path.isdir(tgt): + main.log.fail("target %s<.> is not a readable file<.>, skipping", tgt) continue - original_filesize = over.text.Unit(files.infile.stat().st_size, "o") - main.print("processing %s<.> (%s)" %(tgt, original_filesize), main.print.tl.start) + main.log.note("processing %s<.>", tgt) # -------------------------------------------------- # identify the input file @@ -164,11 +715,12 @@ if __name__ == "__main__": identify_raw = command.identify.get_all_output().decode("utf-8") identify_dict = json.loads(identify_raw) - if main.cfg.probe: + if main.cfg.identify: print(identify_raw) continue info = over.types.ndict() + info.t_start = time.monotonic() try: info.duration = float(identify_dict["format"]["duration"]) @@ -180,10 +732,10 @@ if __name__ == "__main__": amount_as = len(audio_streams) if amount_vs > 1: - main.print('detected %d<.> video streams, picking the "best" one (see man 1 ffmpeg, section STREAM SELECTION)' %(amount_vs), main.print.tl.warn) + main.log.warn('detected %d<.> video streams, picking the "best" one (see man 1 ffmpeg, section STREAM SELECTION)', amount_vs) if amount_as > 1: - main.print('detected %d<.> audio streams, picking the "best" one (see man 1 ffmpeg, section STREAM SELECTION)' %(amount_as), main.print.tl.warn) + main.log.warn('detected %d<.> audio streams, picking the "best" one (see man 1 ffmpeg, section STREAM SELECTION)', amount_as) if video_streams: # ffmpeg picks the stream with the highest pixel count and then the lowest index @@ -201,8 +753,6 @@ if __name__ == "__main__": else: info.video_bitrate = "??<.>" info.pixel_fmt = video["pix_fmt"] - else: - info.video_fps = 30 # faked for progress bars if audio_streams: # ffmpeg picks the stream with the most channels and then the lowest index @@ -218,19 +768,27 @@ if __name__ == "__main__": info.audio_channel_layout = None except: - main.print("exception while reading identify_dict, dump follows", main.print.tl.fail) + main.log.fail("exception while reading identify_dict, dump follows") print(identify_dict) raise + original_filesize = over.text.Unit(files.infile.stat().st_size, "B", base=2) + original_bitrate = over.text.Unit(original_filesize.value * 8 / info.duration, "b/s") + main.log.info(" file: size=%s duration=%s bitrate=%s" %( + original_filesize, + over.text.timestamp_to_hms(info.duration), + original_bitrate + )) + if video_streams: - main.print("video<.>: size=%d<.>x%d<.> px, framerate=%s, codec=%s, bitrate=%s" %(info.video_size_x, info.video_size_y, info.video_fps, info.video_codec, info.video_bitrate)) + main.log.info("video<.>: size=%d<.>x%d<.> px, framerate=%s, codec=%s, bitrate=%s", info.video_size_x, info.video_size_y, info.video_fps, info.video_codec, info.video_bitrate) else: - main.print("video<.>: None<.>", main.print.tl.warn) + main.log.warn("video<.>: none<.>") if audio_streams: - main.print("audio<.>: channels=%d<.>, samplerate=%s, codec=%s, bitrate=%s, language=%s" %(info.audio_channels, info.audio_samplerate, info.audio_codec, info.audio_bitrate, info.audio_language)) + main.log.info("audio<.>: channels=%d<.>, samplerate=%s, codec=%s, bitrate=%s, language=%s", info.audio_channels, info.audio_samplerate, info.audio_codec, info.audio_bitrate, info.audio_language) else: - main.print("audio<.>: None<.>", main.print.tl.warn) + main.log.warn("audio<.>: none<.>") # prepare input stream mapping (which needs to be correct even for the normalization pre-pass) if main.cfg.ffmpeg_map: @@ -245,8 +803,8 @@ if __name__ == "__main__": # -------------------------------------------------- # normalization pre-pass - if audio_streams and main.cfg.normalize and (main.cfg.armed or main.cfg.dump_commands) and (main.cfg.audio not in ["copy", "drop"]): - main.print("running normalization pre-pass") + if audio_streams and main.cfg.normalize and (main.cfg.armed or main.cfg.dump) and (main.cfg.audio not in ["copy", "drop"]): + main.log.begin("running normalization pre-pass") command.normalize_prepass.reset() command.normalize_prepass.INFILE = "file:" + str(files.infile) @@ -271,7 +829,7 @@ if __name__ == "__main__": output_buffer = [] while True: - time.sleep(.25) + time.sleep(.5) out = command.normalize_prepass.get_output() @@ -286,7 +844,7 @@ if __name__ == "__main__": elif out is None: break - pb.end() + pb.end(True) output = b"".join(output_buffer) @@ -295,15 +853,13 @@ if __name__ == "__main__": loudnorm_dict = json.loads(output[output.index(b"{"):].decode("ascii")) info.loudnorm = over.types.ndict({k: aux.float_or_string(v) for k, v in loudnorm_dict.items()}) else: - main.print("unexpected ffmpeg output<.>, dump follows", main.print.tl.fail, end=":\n") + main.log.fail("unexpected ffmpeg output<.>, dump follows") print(output.decode("utf-8")) raise RuntimeError - main.print("detected true peak %.1f dB" %(info.loudnorm.input_tp)) + main.log.done("detected true peak %.1f dB", info.loudnorm.input_tp) - info.normalize_command = command.sub_normalize - info.normalize_command.reset() - info.normalize_command.LOUDNORM_INCANTATION = "loudnorm=i=-23.0:lra=7.0:tp=-2.0:offset=0.0:measured_i=%.02f:measured_lra=%.02f:measured_tp=%.02f:measured_thresh=%.02f:linear=true" %( + info.loudnorm_afilter = "loudnorm=i=-23.0:lra=7.0:tp=-2.0:offset=0.0:measured_i=%.02f:measured_lra=%.02f:measured_tp=%.02f:measured_thresh=%.02f:linear=true" %( info.loudnorm.input_i, info.loudnorm.input_lra, info.loudnorm.input_tp, @@ -311,59 +867,57 @@ if __name__ == "__main__": ) else: - info.normalize_command = None + info.loudnorm_afilter = None # -------------------------------------------------- # main command assembly - encode_cmd = command.encode_generic encode_cmd.reset() - encode_cmd.INFILE = "file:" + str(files.infile) encode_cmd.OUTFILE = files.tmpfile - encode_cmd.FPS = ["-r", main.cfg.fps] if main.cfg.fps else None - encode_cmd.CUT_FROM = ["-ss", main.cfg.cut[0]] if main.cfg.cut else None encode_cmd.CUT_TO = ["-to", main.cfg.cut[1]] if main.cfg.cut else None + encode_cmd.MAP = info.map_command - if main.cfg.audio == "copy": - encode_cmd.AUDIO = command.sub_copy_audio - elif main.cfg.audio == "drop": - encode_cmd.AUDIO = "-an" - elif main.cfg.audio == "pcm": - command.sub_pcm.reset() - command.sub_pcm.NORMALIZE = info.normalize_command + ## + ## audio + ## + if main.cfg.ffmpeg_afilter: + info.afilter_command = command.sub_afilter + info.afilter_command.reset() + info.afilter_command.ARGS = main.cfg.ffmpeg_afilter + else: + info.afilter_command = None + + audio_codec, audio_incantation = audio_codec_router.get_command(main.cfg.audio) + + if audio_codec not in ["drop", "copy"]: + afilters = [] - encode_cmd.AUDIO = command.sub_pcm - elif main.cfg.audio == "vorbis": - command.sub_vorbis.reset() - command.sub_vorbis.QUALITY = main.cfg.audio_quality - command.sub_vorbis.NORMALIZE = info.normalize_command + if main.cfg.ffmpeg_afilter: + afilters.append(main.cfg.ffmpeg_afilter) - encode_cmd.AUDIO = command.sub_vorbis - elif main.cfg.audio == "aac": - command.sub_aac.reset() - command.sub_aac.NORMALIZE = info.normalize_command - - encode_cmd.AUDIO = command.sub_aac - elif main.cfg.audio == "opus": - command.sub_opus.reset() - command.sub_opus.NORMALIZE = info.normalize_command + if info.loudnorm_afilter: + afilters.append(info.loudnorm_afilter) # workaround of https://trac.ffmpeg.org/ticket/5718 - if info.audio_channel_layout == "5.1(side)" or main.cfg.force_5718: - main.print("applying #5718 workaround<.>", main.print.tl.warn) - command.sub_opus.sequence.append("-filter:a") - command.sub_opus.sequence.append("channelmap=channel_layout=5.1") + if audio_codec == "opus" and (info.audio_channel_layout == "5.1(side)" or main.cfg.force_5718): + main.log.warn("applying #5718 workaround<.>") + afilters.append("channelmap=channel_layout=5.1") - encode_cmd.AUDIO = command.sub_opus - elif main.cfg.audio == "mp3": - command.sub_mp3.reset() - command.sub_mp3.NORMALIZE = info.normalize_command - - encode_cmd.AUDIO = command.sub_mp3 + if afilters: + afilter_command = command.sub_afilter + afilter_command.reset() + afilter_command.ARGS = ",".join(afilters) + + audio_incantation.extend(afilter_command.dump()) + encode_cmd.AUDIO = audio_incantation + + ## + ## video + ## if main.cfg.ffmpeg_vfilter: info.vfilter_command = command.sub_vfilter info.vfilter_command.reset() @@ -371,57 +925,36 @@ if __name__ == "__main__": else: info.vfilter_command = None - encode_cmd.MAP = info.map_command + video_codec, video_incantation = video_codec_router.get_command(main.cfg.video) - if main.cfg.video == "copy": - encode_cmd.VIDEO = command.sub_copy_video - elif main.cfg.video == "drop": - encode_cmd.VIDEO = "-vn" - elif main.cfg.video == "vp9": - command.sub_vp9.reset() - command.sub_vp9.QUALITY = main.cfg.video_quality - command.sub_vp9.VFILTER = info.vfilter_command - - encode_cmd.VIDEO = command.sub_vp9 - elif main.cfg.video == "av1": - command.sub_av1.reset() - command.sub_av1.QUALITY = main.cfg.video_quality - command.sub_av1.VFILTER = info.vfilter_command - - encode_cmd.VIDEO = command.sub_av1 - elif main.cfg.video == "x264": - command.sub_x264.reset() - command.sub_x264.QUALITY = main.cfg.video_quality - command.sub_x264.PRESET = main.cfg.video_preset - command.sub_x264.VFILTER = info.vfilter_command - - if info.pixel_fmt in X264_BANNED_PIXFMTS: - main.print("source pixel format %s<.> is incompatible with x264, forcing yuv420p<.>" %(info.pixel_fmt), main.print.tl.warn) - command.sub_x264.PIXFMT = command.force_yuv420p - else: - command.sub_x264.PIXFMT = None - - encode_cmd.VIDEO = command.sub_x264 - elif main.cfg.video == "x265": - command.sub_x265.reset() - command.sub_x265.QUALITY = main.cfg.video_quality - command.sub_x265.PRESET = main.cfg.video_preset - command.sub_x265.VFILTER = info.vfilter_command - - encode_cmd.VIDEO = command.sub_x265 + if info.vfilter_command and video_codec not in ["drop", "copy"]: + video_incantation.extend(info.vfilter_command.dump()) + + if video_codec == "x264" and info.pixel_fmt in ["bgr24", "yuv422p"]: + main.log.warn("source pixel format %s<.> is incompatible with x264, forcing yuv420p<.>", info.pixel_fmt) + video_incantation.extend(command.force_yuv420p.dump()) + + encode_cmd.VIDEO = video_incantation + + ## + ## Subtitles + ## + subtitle_codec, subtitle_incantation = subtitle_codec_router.get_command(main.cfg.subtitle) + + encode_cmd.SUBTITLE = subtitle_incantation # -------------------------------------------------- # run the command iff armed - if main.cfg.dump_commands or main.cfg.armed: + if main.cfg.dump or main.cfg.armed: cmd = " ".join(encode_cmd.dump(pretty=True)) if main.cfg.armed: - main.print("executing %s<.>" %(cmd), main.print.tl.start) + main.log.begin("executing %s<.>", cmd) else: - main.print("will execute %s<.>" %(cmd)) + main.log.info("will execute %s<.>", cmd) else: - main.print("will encode into %s<.>" %(files.tmpfile)) + main.log.info("will encode into %s<.>", files.tmpfile) if main.cfg.armed: pb = over.text.ProgressBar( @@ -431,11 +964,13 @@ if __name__ == "__main__": "unit": "s", "top": int(info.duration), "precision": 1, - "min_width_rate": 9 + "min_width_rate": 9, + "only_prefixes": ["µ", "m", ""] }, "s": { - "unit": "o", # octets are cool - "top": None, # size is unknown at the start but will be estimated during updates + "unit": "B", + "base": 2, + "top": None, # size only becomes known in the subsequent updates "precision": 1, "min_width_raw": 9, "min_width_rate": 11 @@ -446,7 +981,7 @@ if __name__ == "__main__": encode_cmd.run(stderr=True) while True: - time.sleep(.25) + time.sleep(.5) out = encode_cmd.get_output() @@ -456,7 +991,7 @@ if __name__ == "__main__": try: pb.set("a", int(t)) except ZeroDivisionError: - print(out) + pb.set("a", int(1.0)) elif out is None: break @@ -468,14 +1003,21 @@ if __name__ == "__main__": pb.render() - new_filesize = over.text.Unit(files.tmpfile.stat().st_size, "o") + new_filesize = over.text.Unit(files.tmpfile.stat().st_size, "B", base=2) + new_bitrate = over.text.Unit(new_filesize.value * 8 / info.duration, "b/s") - pb.end() + pb.end(True) if encode_cmd.returncode == 0: - main.print("encoding finished: %s -> %s" %(original_filesize, new_filesize), main.print.tl.done) + main.log.done("success, %s (%s) -> %s (%s) in %s" %( + original_filesize, + original_bitrate, + new_filesize, + new_bitrate, + over.text.timestamp_to_hms(time.monotonic() - info.t_start) + )) else: - main.print("encoding failed<.>, ffmpeg returned %d<.>" %(encode_cmd.returncode), main.print.tl.fail) + main.log.fail("encoding failed<.>, ffmpeg returned %d<.>", encode_cmd.returncode) raise RuntimeError # -------------------------------------------------- @@ -492,44 +1034,44 @@ if __name__ == "__main__": cmd = " ".join(command.merge_chapters.dump(pretty=True)) if main.cfg.armed: - main.print("moving %s<.> -> %s<.>" %(files.tmpfile, files.tmpfile_nochapters), main.print.tl.start) + main.log.info("moving %s<.> -> %s<.>", files.tmpfile, files.tmpfile_nochapters) files.tmpfile.rename(files.tmpfile_nochapters) - main.print("executing %s<.>" %(cmd), main.print.tl.start) + main.log.info("executing %s<.>", cmd) command.merge_chapters.run() _ = command.merge_chapters.get_all_output() - main.print("deleting %s<.>" %(files.tmpfile_nochapters)) + main.log.info("deleting %s<.>", files.tmpfile_nochapters) files.tmpfile_nochapters.unlink() else: - main.print("will move %s<.> -> %s<.>" %(files.tmpfile, files.tmpfile_nochapters)) - main.print("will execute %s<.>" %(cmd)) - main.print("will delete %s<.>" %(files.tmpfile_nochapters)) + main.log.info("will move %s<.> -> %s<.>", files.tmpfile, files.tmpfile_nochapters) + main.log.info("will execute %s<.>", cmd) + main.log.info("will delete %s<.>", files.tmpfile_nochapters) else: - main.print("will add chapters from %s<.>" %(main.cfg.chapters)) + main.print("will add chapters from %s<.>", main.cfg.chapters) # -------------------------------------------------- # shuffle files around - if main.cfg.move_source: - move_to_dir = pathlib.Path(main.cfg.move_source) + if main.cfg.move: + move_to_dir = pathlib.Path(main.cfg.move) if not move_to_dir.is_dir(): if main.cfg.armed: - main.print("creating directory %s<.>" %(move_to_dir), main.print.tl.start) + main.log.info("creating directory %s<.>", move_to_dir) move_to_dir.mkdir() else: - main.print("will create directory %s<.>" %(move_to_dir)) + main.log.info("will create directory %s<.>", move_to_dir) if files.move_infile_to: if main.cfg.armed: - main.print("moving %s<.> -> %s<.>" %(files.infile, files.move_infile_to), main.print.tl.start) + main.log.info("moving %s<.> -> %s<.>", files.infile, files.move_infile_to) files.infile.rename(files.move_infile_to) else: - main.print("will move %s<.> -> %s<.>" %(files.infile, files.move_infile_to)) + main.log.info("will move %s<.> -> %s<.>", files.infile, files.move_infile_to) if main.cfg.armed: - main.print("moving %s<.> -> %s<.>" %(files.tmpfile, files.outfile), main.print.tl.start) + main.log.info("moving %s<.> -> %s<.>", files.tmpfile, files.outfile) files.tmpfile.rename(files.outfile) else: - main.print("will move %s<.> -> %s<.>" %(files.tmpfile, files.outfile)) + main.log.info("will move %s<.> -> %s<.>", files.tmpfile, files.outfile) diff --git a/version.py b/version.py index a6ea6dd..0505248 100644 --- a/version.py +++ b/version.py @@ -1,8 +1,8 @@ #! /usr/bin/env python3 # encoding: utf-8 -major = 1 # VERSION_MAJOR_IDENTIFIER -minor = 116 # VERSION_MINOR_IDENTIFIER -# VERSION_LAST_MM 1.116 +major = 2 # VERSION_MAJOR_IDENTIFIER +minor = 0 # VERSION_MINOR_IDENTIFIER +# VERSION_LAST_MM 2.0 patch = 0 # VERSION_PATCH_IDENTIFIER str = ".".join(str(v) for v in (major, minor, patch))