From 8eb2293808ea1f0685a9bbb6b76c60dfb502d8b7 Mon Sep 17 00:00:00 2001 From: asonix Date: Fri, 30 Sep 2022 19:38:11 -0500 Subject: [PATCH] Enable selecting video and audio codecs for uploaded media --- README.md | 8 ++- defaults.toml | 2 + pict-rs.toml | 20 ++++++++ src/config.rs | 4 +- src/config/commandline.rs | 20 ++++++-- src/config/defaults.rs | 4 +- src/config/file.rs | 6 ++- src/config/primitives.rs | 42 +++++++++++++++ src/details.rs | 24 ++++++++- src/ffmpeg.rs | 104 +++++++++++++++++++++++++++++--------- src/generate.rs | 10 +++- src/ingest.rs | 4 +- src/main.rs | 9 ++-- src/queue/process.rs | 4 ++ src/validate.rs | 36 ++++++++++--- 15 files changed, 252 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 09a886b..8e6a209 100644 --- a/README.md +++ b/README.md @@ -80,9 +80,13 @@ Options: --media-max-frame-count The maximum number of frames allowed for uploaded GIF and MP4s --media-enable-silent-video - Whether to enable GIF and silent MP4 uploads [possible values: true, false] + Whether to enable GIF and silent video uploads [possible values: true, false] --media-enable-full-video - Whether to enable full MP4 uploads [possible values: true, false] + Whether to enable full video uploads [possible values: true, false] + --media-video-codec + Enforce a specific video codec for uploaded videos [possible values: h264, h265, av1, vp8, vp9] + --media-audio-codec + Enforce a specific audio codec for uploaded videos [possible values: aac, opus, vorbis] --media-filters Which media filters should be enabled on the `process` endpoint --media-format diff --git a/defaults.toml b/defaults.toml index c71f134..82c4b44 100644 --- a/defaults.toml +++ b/defaults.toml @@ -1,6 +1,7 @@ [server] address = '0.0.0.0:8080' worker_id = 'pict-rs-1' + [tracing.logging] format = 'normal' targets = 'warn,tracing_actix_web=info,actix_server=info,actix_web=info' @@ -23,6 +24,7 @@ max_file_size = 40 max_frame_count = 900 enable_silent_video = true enable_full_video = false +video_codec = 'h264' filters = ['blur', 'crop', 'identity', 'resize', 'thumbnail'] skip_validate_imports = false cache_duration = 168 diff --git a/pict-rs.toml b/pict-rs.toml index 9371d9e..4511426 100644 --- a/pict-rs.toml +++ b/pict-rs.toml @@ -159,6 +159,26 @@ enable_silent_video = true # default: false enable_full_video = false +## Optional: set the default video codec +# environment variable: PICTRS__MEDIA__VIDEO_CODEC +# default: h264 +# +# available options: av1, h264, h265, vp8, vp9 +# this setting does nothing if video is not enabled +video_codec = "h264" + +## Optional: set the default audio codec +# environment variable: PICTRS__MEDIA__AUDIO_CODEC +# default: empty +# +# available options: aac, opus, vorbis +# The audio codec is automatically selected based on video codec, but can be overriden +# av1, vp8, and vp9 map to opus +# h264 and h265 map to aac +# vorbis is not default for any codec +# this setting does nothing if full video is not enabled +audio_codec = "aac" + ## Optional: set allowed filters for image processing # environment variable: PICTRS__MEDIA__FILTERS # default: ['blur', 'crop', 'identity', 'resize', 'thumbnail'] diff --git a/src/config.rs b/src/config.rs index 93e8e6a..495b0fd 100644 --- a/src/config.rs +++ b/src/config.rs @@ -11,7 +11,9 @@ use defaults::Defaults; pub(crate) use commandline::Operation; pub(crate) use file::{ConfigFile as Configuration, OpenTelemetry, Repo, Sled, Tracing}; -pub(crate) use primitives::{Filesystem, ImageFormat, LogFormat, ObjectStorage, Store}; +pub(crate) use primitives::{ + AudioCodec, Filesystem, ImageFormat, LogFormat, ObjectStorage, Store, VideoCodec, +}; pub(crate) fn configure() -> color_eyre::Result<(Configuration, Operation)> { let Output { diff --git a/src/config/commandline.rs b/src/config/commandline.rs index 9177af2..dc538a4 100644 --- a/src/config/commandline.rs +++ b/src/config/commandline.rs @@ -1,5 +1,5 @@ use crate::{ - config::primitives::{ImageFormat, LogFormat, Targets}, + config::primitives::{AudioCodec, ImageFormat, LogFormat, Targets, VideoCodec}, serde_str::Serde, }; use clap::{Parser, Subcommand}; @@ -54,6 +54,8 @@ impl Args { media_max_frame_count, media_enable_silent_video, media_enable_full_video, + media_video_codec, + media_audio_codec, media_filters, media_format, media_cache_duration, @@ -74,6 +76,8 @@ impl Args { max_frame_count: media_max_frame_count, enable_silent_video: media_enable_silent_video, enable_full_video: media_enable_full_video, + video_codec: media_video_codec, + audio_codec: media_audio_codec, filters: media_filters, format: media_format, cache_duration: media_cache_duration, @@ -322,6 +326,10 @@ struct Media { #[serde(skip_serializing_if = "Option::is_none")] enable_full_video: Option, #[serde(skip_serializing_if = "Option::is_none")] + video_codec: Option, + #[serde(skip_serializing_if = "Option::is_none")] + audio_codec: Option, + #[serde(skip_serializing_if = "Option::is_none")] filters: Option>, #[serde(skip_serializing_if = "Option::is_none")] format: Option, @@ -423,12 +431,18 @@ struct Run { /// The maximum number of frames allowed for uploaded GIF and MP4s. #[arg(long)] media_max_frame_count: Option, - /// Whether to enable GIF and silent MP4 uploads + /// Whether to enable GIF and silent video uploads #[arg(long)] media_enable_silent_video: Option, - /// Whether to enable full MP4 uploads + /// Whether to enable full video uploads #[arg(long)] media_enable_full_video: Option, + /// Enforce a specific video codec for uploaded videos + #[arg(long)] + media_video_codec: Option, + /// Enforce a specific audio codec for uploaded videos + #[arg(long)] + media_audio_codec: Option, /// Which media filters should be enabled on the `process` endpoint #[arg(long)] media_filters: Option>, diff --git a/src/config/defaults.rs b/src/config/defaults.rs index 7873907..2e300f3 100644 --- a/src/config/defaults.rs +++ b/src/config/defaults.rs @@ -1,5 +1,5 @@ use crate::{ - config::primitives::{LogFormat, Targets}, + config::primitives::{LogFormat, Targets, VideoCodec}, serde_str::Serde, }; use std::{net::SocketAddr, path::PathBuf}; @@ -68,6 +68,7 @@ struct MediaDefaults { max_frame_count: usize, enable_silent_video: bool, enable_full_video: bool, + video_codec: VideoCodec, filters: Vec, skip_validate_imports: bool, cache_duration: i64, @@ -155,6 +156,7 @@ impl Default for MediaDefaults { max_frame_count: 900, enable_silent_video: true, enable_full_video: false, + video_codec: VideoCodec::H264, filters: vec![ "blur".into(), "crop".into(), diff --git a/src/config/file.rs b/src/config/file.rs index 9a43206..242067d 100644 --- a/src/config/file.rs +++ b/src/config/file.rs @@ -1,5 +1,5 @@ use crate::{ - config::primitives::{ImageFormat, LogFormat, Store, Targets}, + config::primitives::{AudioCodec, ImageFormat, LogFormat, Store, Targets, VideoCodec}, serde_str::Serde, }; use once_cell::sync::OnceCell; @@ -104,6 +104,10 @@ pub(crate) struct Media { pub(crate) enable_full_video: bool, + pub(crate) video_codec: VideoCodec, + + pub(crate) audio_codec: Option, + pub(crate) filters: BTreeSet, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/src/config/primitives.rs b/src/config/primitives.rs index 531ab5d..6e2be94 100644 --- a/src/config/primitives.rs +++ b/src/config/primitives.rs @@ -45,6 +45,48 @@ pub(crate) enum ImageFormat { Png, } +#[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde::Deserialize, + serde::Serialize, + ValueEnum, +)] +#[serde(rename_all = "snake_case")] +pub(crate) enum VideoCodec { + H264, + H265, + Av1, + Vp8, + Vp9, +} + +#[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + serde::Deserialize, + serde::Serialize, + ValueEnum, +)] +#[serde(rename_all = "snake_case")] +pub(crate) enum AudioCodec { + Aac, + Opus, + Vorbis, +} + #[derive(Clone, Debug)] pub(crate) struct Targets { pub(crate) targets: tracing_subscriber::filter::Targets, diff --git a/src/details.rs b/src/details.rs index 91a696c..e310ff4 100644 --- a/src/details.rs +++ b/src/details.rs @@ -1,4 +1,10 @@ -use crate::{error::Error, magick::ValidInputType, serde_str::Serde, store::Store}; +use crate::{ + error::Error, + ffmpeg::InputFormat, + magick::{video_mp4, video_webm, ValidInputType}, + serde_str::Serde, + store::Store, +}; use actix_web::web; #[derive(Copy, Clone, Debug, serde::Deserialize, serde::Serialize)] @@ -93,6 +99,22 @@ impl Details { pub(crate) fn system_time(&self) -> std::time::SystemTime { self.created_at.into() } + + pub(crate) fn to_input_format(&self) -> Option { + if *self.content_type == mime::IMAGE_GIF { + return Some(InputFormat::Gif); + } + + if *self.content_type == video_mp4() { + return Some(InputFormat::Mp4); + } + + if *self.content_type == video_webm() { + return Some(InputFormat::Webm); + } + + None + } } impl From for std::time::SystemTime { diff --git a/src/ffmpeg.rs b/src/ffmpeg.rs index 2a34308..64f219a 100644 --- a/src/ffmpeg.rs +++ b/src/ffmpeg.rs @@ -1,4 +1,5 @@ use crate::{ + config::{AudioCodec, VideoCodec}, error::{Error, UploadError}, magick::{Details, ValidInputType}, process::Process, @@ -8,25 +9,31 @@ use actix_web::web::Bytes; use tokio::io::{AsyncRead, AsyncReadExt}; use tracing::instrument; -#[derive(Copy, Clone, Debug)] +#[derive(Clone, Copy, Debug)] pub(crate) enum InputFormat { Gif, Mp4, Webm, } -#[derive(Copy, Clone, Debug)] +#[derive(Clone, Copy, Debug)] +pub(crate) enum OutputFormat { + Mp4, + Webm, +} + +#[derive(Clone, Copy, Debug)] pub(crate) enum ThumbnailFormat { Jpeg, // Webp, } impl InputFormat { - fn to_ext(self) -> &'static str { + const fn to_ext(self) -> &'static str { match self { - InputFormat::Gif => ".gif", - InputFormat::Mp4 => ".mp4", - InputFormat::Webm => ".webm", + Self::Gif => ".gif", + Self::Mp4 => ".mp4", + Self::Webm => ".webm", } } @@ -40,23 +47,69 @@ impl InputFormat { } impl ThumbnailFormat { - fn as_codec(self) -> &'static str { + const fn as_ffmpeg_codec(self) -> &'static str { match self { - ThumbnailFormat::Jpeg => "mjpeg", - // ThumbnailFormat::Webp => "webp", + Self::Jpeg => "mjpeg", + // Self::Webp => "webp", } } - fn to_ext(self) -> &'static str { + const fn to_ext(self) -> &'static str { match self { - ThumbnailFormat::Jpeg => ".jpeg", + Self::Jpeg => ".jpeg", + // Self::Webp => ".webp", } } - fn as_format(self) -> &'static str { + const fn as_ffmpeg_format(self) -> &'static str { match self { - ThumbnailFormat::Jpeg => "image2", - // ThumbnailFormat::Webp => "webp", + Self::Jpeg => "image2", + // Self::Webp => "webp", + } + } +} + +impl OutputFormat { + const fn to_ffmpeg_format(self) -> &'static str { + match self { + Self::Mp4 => "mp4", + Self::Webm => "webm", + } + } + + const fn default_audio_codec(self) -> AudioCodec { + match self { + Self::Mp4 => AudioCodec::Aac, + Self::Webm => AudioCodec::Opus, + } + } +} + +impl VideoCodec { + const fn to_output_format(self) -> OutputFormat { + match self { + Self::H264 | Self::H265 => OutputFormat::Mp4, + Self::Av1 | Self::Vp8 | Self::Vp9 => OutputFormat::Webm, + } + } + + const fn to_ffmpeg_codec(self) -> &'static str { + match self { + Self::Av1 => "av1", + Self::H264 => "h264", + Self::H265 => "hevc", + Self::Vp8 => "vp8", + Self::Vp9 => "vp9", + } + } +} + +impl AudioCodec { + const fn to_ffmpeg_codec(self) -> &'static str { + match self { + Self::Aac => "aac", + Self::Opus => "opus", + Self::Vorbis => "vorbis", } } } @@ -184,11 +237,13 @@ fn parse_details_inner( }) } -#[tracing::instrument(name = "Convert to Mp4", skip(input))] -pub(crate) async fn to_mp4_bytes( +#[tracing::instrument(name = "Transcode video", skip(input))] +pub(crate) async fn trancsocde_bytes( input: Bytes, input_format: InputFormat, permit_audio: bool, + video_codec: VideoCodec, + audio_codec: Option, ) -> Result { let input_file = crate::tmp_file::tmp_file(Some(input_format.to_ext())); let input_file_str = input_file.to_str().ok_or(UploadError::Path)?; @@ -202,6 +257,9 @@ pub(crate) async fn to_mp4_bytes( tmp_one.write_from_bytes(input).await?; tmp_one.close().await?; + let output_format = video_codec.to_output_format(); + let audio_codec = audio_codec.unwrap_or(output_format.default_audio_codec()); + let process = if permit_audio { Process::run( "ffmpeg", @@ -213,11 +271,11 @@ pub(crate) async fn to_mp4_bytes( "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-c:a", - "aac", + audio_codec.to_ffmpeg_codec(), "-c:v", - "h264", + video_codec.to_ffmpeg_codec(), "-f", - "mp4", + output_format.to_ffmpeg_format(), output_file_str, ], )? @@ -233,9 +291,9 @@ pub(crate) async fn to_mp4_bytes( "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-an", "-c:v", - "h264", + video_codec.to_ffmpeg_codec(), "-f", - "mp4", + output_format.to_ffmpeg_format(), output_file_str, ], )? @@ -281,9 +339,9 @@ pub(crate) async fn thumbnail( "-vframes", "1", "-codec", - format.as_codec(), + format.as_ffmpeg_codec(), "-f", - format.as_format(), + format.as_ffmpeg_format(), output_file_str, ], )?; diff --git a/src/generate.rs b/src/generate.rs index ae7ecb9..73d874a 100644 --- a/src/generate.rs +++ b/src/generate.rs @@ -20,6 +20,8 @@ pub(crate) async fn generate( alias: Alias, thumbnail_path: PathBuf, thumbnail_args: Vec, + input_format: Option, + thumbnail_format: Option, hash: R::Bytes, ) -> Result<(Details, Bytes), Error> { let process_fut = process( @@ -29,6 +31,8 @@ pub(crate) async fn generate( alias, thumbnail_path.clone(), thumbnail_args, + input_format, + thumbnail_format, hash.clone(), ); @@ -46,6 +50,8 @@ async fn process( alias: Alias, thumbnail_path: PathBuf, thumbnail_args: Vec, + input_format: Option, + thumbnail_format: Option, hash: R::Bytes, ) -> Result<(Details, Bytes), Error> { let permit = crate::PROCESS_SEMAPHORE.acquire().await; @@ -60,8 +66,8 @@ async fn process( let reader = crate::ffmpeg::thumbnail( store.clone(), identifier, - InputFormat::Mp4, - ThumbnailFormat::Jpeg, + input_format.unwrap_or(InputFormat::Mp4), + thumbnail_format.unwrap_or(ThumbnailFormat::Jpeg), ) .await?; let motion_identifier = store.save_async_read(reader).await?; diff --git a/src/ingest.rs b/src/ingest.rs index 04c3562..fc3fd14 100644 --- a/src/ingest.rs +++ b/src/ingest.rs @@ -70,11 +70,13 @@ where let bytes = aggregate(stream).await?; tracing::debug!("Validating bytes"); - let (input_type, validated_reader) = crate::validate::validate_image_bytes( + let (input_type, validated_reader) = crate::validate::validate_bytes( bytes, CONFIG.media.format, CONFIG.media.enable_silent_video, CONFIG.media.enable_full_video, + CONFIG.media.video_codec, + CONFIG.media.audio_codec, should_validate, ) .await?; diff --git a/src/main.rs b/src/main.rs index ed1f0ec..c9b5d2a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -48,8 +48,6 @@ mod stream; mod tmp_file; mod validate; -use crate::magick::ValidInputType; - use self::{ backgrounded::Backgrounded, config::{Configuration, ImageFormat, Operation}, @@ -58,7 +56,7 @@ use self::{ error::{Error, UploadError}, ingest::Session, init_tracing::init_tracing, - magick::details_hint, + magick::{details_hint, ValidInputType}, middleware::{Deadline, Internal}, queue::queue_generate, repo::{ @@ -597,6 +595,7 @@ async fn process( let path_string = thumbnail_path.to_string_lossy().to_string(); let hash = repo.hash(&alias).await?; + let identifier_opt = repo .variant_identifier::(hash.clone(), path_string) .await?; @@ -624,6 +623,8 @@ async fn process( return ranged_file_resp(&store, identifier, range, details).await; } + let original_details = ensure_details(&repo, &store, &alias).await?; + let (details, bytes) = generate::generate( &repo, &store, @@ -631,6 +632,8 @@ async fn process( alias, thumbnail_path, thumbnail_args, + original_details.to_input_format(), + None, hash, ) .await?; diff --git a/src/queue/process.rs b/src/queue/process.rs index 1ae9877..7b7ae42 100644 --- a/src/queue/process.rs +++ b/src/queue/process.rs @@ -149,6 +149,8 @@ async fn generate( return Ok(()); } + let original_details = crate::ensure_details(repo, store, &source).await?; + crate::generate::generate( repo, store, @@ -156,6 +158,8 @@ async fn generate( source, process_path, process_args, + original_details.to_input_format(), + None, hash, ) .await?; diff --git a/src/validate.rs b/src/validate.rs index 91ecce4..4e030d1 100644 --- a/src/validate.rs +++ b/src/validate.rs @@ -1,5 +1,5 @@ use crate::{ - config::ImageFormat, + config::{AudioCodec, ImageFormat, VideoCodec}, either::Either, error::{Error, UploadError}, ffmpeg::InputFormat, @@ -36,12 +36,14 @@ impl AsyncRead for UnvalidatedBytes { } } -#[instrument(name = "Validate image", skip(bytes))] -pub(crate) async fn validate_image_bytes( +#[instrument(name = "Validate media", skip(bytes))] +pub(crate) async fn validate_bytes( bytes: Bytes, prescribed_format: Option, enable_silent_video: bool, enable_full_video: bool, + video_codec: VideoCodec, + audio_codec: Option, validate: bool, ) -> Result<(ValidInputType, impl AsyncRead + Unpin), Error> { let input_type = @@ -63,7 +65,14 @@ pub(crate) async fn validate_image_bytes( Ok(( ValidInputType::Mp4, Either::right(Either::left( - crate::ffmpeg::to_mp4_bytes(bytes, InputFormat::Gif, false).await?, + crate::ffmpeg::trancsocde_bytes( + bytes, + InputFormat::Gif, + false, + video_codec, + audio_codec, + ) + .await?, )), )) } @@ -74,7 +83,14 @@ pub(crate) async fn validate_image_bytes( Ok(( ValidInputType::Mp4, Either::right(Either::left( - crate::ffmpeg::to_mp4_bytes(bytes, InputFormat::Mp4, enable_full_video).await?, + crate::ffmpeg::trancsocde_bytes( + bytes, + InputFormat::Mp4, + enable_full_video, + video_codec, + audio_codec, + ) + .await?, )), )) } @@ -85,8 +101,14 @@ pub(crate) async fn validate_image_bytes( Ok(( ValidInputType::Mp4, Either::right(Either::left( - crate::ffmpeg::to_mp4_bytes(bytes, InputFormat::Webm, enable_full_video) - .await?, + crate::ffmpeg::trancsocde_bytes( + bytes, + InputFormat::Webm, + enable_full_video, + video_codec, + audio_codec, + ) + .await?, )), )) }