Enable selecting video and audio codecs for uploaded media

README.md

@@ -80,9 +80,13 @@ Options:
       --media-max-frame-count <MEDIA_MAX_FRAME_COUNT>
           The maximum number of frames allowed for uploaded GIF and MP4s
       --media-enable-silent-video <MEDIA_ENABLE_SILENT_VIDEO>
-          Whether to enable GIF and silent MP4 uploads [possible values: true, false]
+          Whether to enable GIF and silent video uploads [possible values: true, false]
       --media-enable-full-video <MEDIA_ENABLE_FULL_VIDEO>
-          Whether to enable full MP4 uploads [possible values: true, false]
+          Whether to enable full video uploads [possible values: true, false]
+      --media-video-codec <MEDIA_VIDEO_CODEC>
+          Enforce a specific video codec for uploaded videos [possible values: h264, h265, av1, vp8, vp9]
+      --media-audio-codec <MEDIA_AUDIO_CODEC>
+          Enforce a specific audio codec for uploaded videos [possible values: aac, opus, vorbis]
       --media-filters <MEDIA_FILTERS>
           Which media filters should be enabled on the `process` endpoint
       --media-format <MEDIA_FORMAT>

defaults.toml

@@ -1,6 +1,7 @@
 [server]
 address = '0.0.0.0:8080'
 worker_id = 'pict-rs-1'
+
 [tracing.logging]
 format = 'normal'
 targets = 'warn,tracing_actix_web=info,actix_server=info,actix_web=info'
@@ -23,6 +24,7 @@ max_file_size = 40
 max_frame_count = 900
 enable_silent_video = true
 enable_full_video = false
+video_codec = 'h264'
 filters = ['blur', 'crop', 'identity', 'resize', 'thumbnail']
 skip_validate_imports = false
 cache_duration = 168

pict-rs.toml

@@ -159,6 +159,26 @@ enable_silent_video = true
 # default: false
 enable_full_video = false
 
+## Optional: set the default video codec
+# environment variable: PICTRS__MEDIA__VIDEO_CODEC
+# default: h264
+#
+# available options: av1, h264, h265, vp8, vp9
+# this setting does nothing if video is not enabled
+video_codec = "h264"
+
+## Optional: set the default audio codec
+# environment variable: PICTRS__MEDIA__AUDIO_CODEC
+# default: empty
+#
+# available options: aac, opus, vorbis
+# The audio codec is automatically selected based on video codec, but can be overriden
+# av1, vp8, and vp9 map to opus
+# h264 and h265 map to aac
+# vorbis is not default for any codec
+# this setting does nothing if full video is not enabled
+audio_codec = "aac"
+
 ## Optional: set allowed filters for image processing
 # environment variable: PICTRS__MEDIA__FILTERS
 # default: ['blur', 'crop', 'identity', 'resize', 'thumbnail']

src/config.rs

@@ -11,7 +11,9 @@ use defaults::Defaults;
 
 pub(crate) use commandline::Operation;
 pub(crate) use file::{ConfigFile as Configuration, OpenTelemetry, Repo, Sled, Tracing};
-pub(crate) use primitives::{Filesystem, ImageFormat, LogFormat, ObjectStorage, Store};
+pub(crate) use primitives::{
+    AudioCodec, Filesystem, ImageFormat, LogFormat, ObjectStorage, Store, VideoCodec,
+};
 
 pub(crate) fn configure() -> color_eyre::Result<(Configuration, Operation)> {
     let Output {

src/config/commandline.rs

@@ -1,5 +1,5 @@
 use crate::{
-    config::primitives::{ImageFormat, LogFormat, Targets},
+    config::primitives::{AudioCodec, ImageFormat, LogFormat, Targets, VideoCodec},
     serde_str::Serde,
 };
 use clap::{Parser, Subcommand};
@@ -54,6 +54,8 @@ impl Args {
                 media_max_frame_count,
                 media_enable_silent_video,
                 media_enable_full_video,
+                media_video_codec,
+                media_audio_codec,
                 media_filters,
                 media_format,
                 media_cache_duration,
@@ -74,6 +76,8 @@ impl Args {
                     max_frame_count: media_max_frame_count,
                     enable_silent_video: media_enable_silent_video,
                     enable_full_video: media_enable_full_video,
+                    video_codec: media_video_codec,
+                    audio_codec: media_audio_codec,
                     filters: media_filters,
                     format: media_format,
                     cache_duration: media_cache_duration,
@@ -322,6 +326,10 @@ struct Media {
     #[serde(skip_serializing_if = "Option::is_none")]
     enable_full_video: Option<bool>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    video_codec: Option<VideoCodec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    audio_codec: Option<AudioCodec>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     filters: Option<Vec<String>>,
     #[serde(skip_serializing_if = "Option::is_none")]
     format: Option<ImageFormat>,
@@ -423,12 +431,18 @@ struct Run {
     /// The maximum number of frames allowed for uploaded GIF and MP4s.
     #[arg(long)]
     media_max_frame_count: Option<usize>,
-    /// Whether to enable GIF and silent MP4 uploads
+    /// Whether to enable GIF and silent video uploads
     #[arg(long)]
     media_enable_silent_video: Option<bool>,
-    /// Whether to enable full MP4 uploads
+    /// Whether to enable full video uploads
     #[arg(long)]
     media_enable_full_video: Option<bool>,
+    /// Enforce a specific video codec for uploaded videos
+    #[arg(long)]
+    media_video_codec: Option<VideoCodec>,
+    /// Enforce a specific audio codec for uploaded videos
+    #[arg(long)]
+    media_audio_codec: Option<AudioCodec>,
     /// Which media filters should be enabled on the `process` endpoint
     #[arg(long)]
     media_filters: Option<Vec<String>>,

src/config/defaults.rs

@@ -1,5 +1,5 @@
 use crate::{
-    config::primitives::{LogFormat, Targets},
+    config::primitives::{LogFormat, Targets, VideoCodec},
     serde_str::Serde,
 };
 use std::{net::SocketAddr, path::PathBuf};
@@ -68,6 +68,7 @@ struct MediaDefaults {
     max_frame_count: usize,
     enable_silent_video: bool,
     enable_full_video: bool,
+    video_codec: VideoCodec,
     filters: Vec<String>,
     skip_validate_imports: bool,
     cache_duration: i64,
@@ -155,6 +156,7 @@ impl Default for MediaDefaults {
             max_frame_count: 900,
             enable_silent_video: true,
             enable_full_video: false,
+            video_codec: VideoCodec::H264,
             filters: vec![
                 "blur".into(),
                 "crop".into(),

src/config/file.rs

@@ -1,5 +1,5 @@
 use crate::{
-    config::primitives::{ImageFormat, LogFormat, Store, Targets},
+    config::primitives::{AudioCodec, ImageFormat, LogFormat, Store, Targets, VideoCodec},
     serde_str::Serde,
 };
 use once_cell::sync::OnceCell;
@@ -104,6 +104,10 @@ pub(crate) struct Media {
 
     pub(crate) enable_full_video: bool,
 
+    pub(crate) video_codec: VideoCodec,
+
+    pub(crate) audio_codec: Option<AudioCodec>,
+
     pub(crate) filters: BTreeSet<String>,
 
     #[serde(skip_serializing_if = "Option::is_none")]

src/config/primitives.rs

@@ -45,6 +45,48 @@ pub(crate) enum ImageFormat {
     Png,
 }
 
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Hash,
+    serde::Deserialize,
+    serde::Serialize,
+    ValueEnum,
+)]
+#[serde(rename_all = "snake_case")]
+pub(crate) enum VideoCodec {
+    H264,
+    H265,
+    Av1,
+    Vp8,
+    Vp9,
+}
+
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Hash,
+    serde::Deserialize,
+    serde::Serialize,
+    ValueEnum,
+)]
+#[serde(rename_all = "snake_case")]
+pub(crate) enum AudioCodec {
+    Aac,
+    Opus,
+    Vorbis,
+}
+
 #[derive(Clone, Debug)]
 pub(crate) struct Targets {
     pub(crate) targets: tracing_subscriber::filter::Targets,

src/details.rs

@@ -1,4 +1,10 @@
-use crate::{error::Error, magick::ValidInputType, serde_str::Serde, store::Store};
+use crate::{
+    error::Error,
+    ffmpeg::InputFormat,
+    magick::{video_mp4, video_webm, ValidInputType},
+    serde_str::Serde,
+    store::Store,
+};
 use actix_web::web;
 
 #[derive(Copy, Clone, Debug, serde::Deserialize, serde::Serialize)]
@@ -93,6 +99,22 @@ impl Details {
     pub(crate) fn system_time(&self) -> std::time::SystemTime {
         self.created_at.into()
     }
+
+    pub(crate) fn to_input_format(&self) -> Option<InputFormat> {
+        if *self.content_type == mime::IMAGE_GIF {
+            return Some(InputFormat::Gif);
+        }
+
+        if *self.content_type == video_mp4() {
+            return Some(InputFormat::Mp4);
+        }
+
+        if *self.content_type == video_webm() {
+            return Some(InputFormat::Webm);
+        }
+
+        None
+    }
 }
 
 impl From<MaybeHumanDate> for std::time::SystemTime {

src/ffmpeg.rs

@@ -1,4 +1,5 @@
 use crate::{
+    config::{AudioCodec, VideoCodec},
     error::{Error, UploadError},
     magick::{Details, ValidInputType},
     process::Process,
@@ -8,25 +9,31 @@ use actix_web::web::Bytes;
 use tokio::io::{AsyncRead, AsyncReadExt};
 use tracing::instrument;
 
-#[derive(Copy, Clone, Debug)]
+#[derive(Clone, Copy, Debug)]
 pub(crate) enum InputFormat {
     Gif,
     Mp4,
     Webm,
 }
 
-#[derive(Copy, Clone, Debug)]
+#[derive(Clone, Copy, Debug)]
+pub(crate) enum OutputFormat {
+    Mp4,
+    Webm,
+}
+
+#[derive(Clone, Copy, Debug)]
 pub(crate) enum ThumbnailFormat {
     Jpeg,
     // Webp,
 }
 
 impl InputFormat {
-    fn to_ext(self) -> &'static str {
+    const fn to_ext(self) -> &'static str {
         match self {
-            InputFormat::Gif => ".gif",
-            InputFormat::Mp4 => ".mp4",
-            InputFormat::Webm => ".webm",
+            Self::Gif => ".gif",
+            Self::Mp4 => ".mp4",
+            Self::Webm => ".webm",
         }
     }
 
@@ -40,23 +47,69 @@ impl InputFormat {
 }
 
 impl ThumbnailFormat {
-    fn as_codec(self) -> &'static str {
+    const fn as_ffmpeg_codec(self) -> &'static str {
         match self {
-            ThumbnailFormat::Jpeg => "mjpeg",
-            // ThumbnailFormat::Webp => "webp",
+            Self::Jpeg => "mjpeg",
+            // Self::Webp => "webp",
         }
     }
 
-    fn to_ext(self) -> &'static str {
+    const fn to_ext(self) -> &'static str {
         match self {
-            ThumbnailFormat::Jpeg => ".jpeg",
+            Self::Jpeg => ".jpeg",
+            // Self::Webp => ".webp",
         }
     }
 
-    fn as_format(self) -> &'static str {
+    const fn as_ffmpeg_format(self) -> &'static str {
         match self {
-            ThumbnailFormat::Jpeg => "image2",
-            // ThumbnailFormat::Webp => "webp",
+            Self::Jpeg => "image2",
+            // Self::Webp => "webp",
+        }
+    }
+}
+
+impl OutputFormat {
+    const fn to_ffmpeg_format(self) -> &'static str {
+        match self {
+            Self::Mp4 => "mp4",
+            Self::Webm => "webm",
+        }
+    }
+
+    const fn default_audio_codec(self) -> AudioCodec {
+        match self {
+            Self::Mp4 => AudioCodec::Aac,
+            Self::Webm => AudioCodec::Opus,
+        }
+    }
+}
+
+impl VideoCodec {
+    const fn to_output_format(self) -> OutputFormat {
+        match self {
+            Self::H264 | Self::H265 => OutputFormat::Mp4,
+            Self::Av1 | Self::Vp8 | Self::Vp9 => OutputFormat::Webm,
+        }
+    }
+
+    const fn to_ffmpeg_codec(self) -> &'static str {
+        match self {
+            Self::Av1 => "av1",
+            Self::H264 => "h264",
+            Self::H265 => "hevc",
+            Self::Vp8 => "vp8",
+            Self::Vp9 => "vp9",
+        }
+    }
+}
+
+impl AudioCodec {
+    const fn to_ffmpeg_codec(self) -> &'static str {
+        match self {
+            Self::Aac => "aac",
+            Self::Opus => "opus",
+            Self::Vorbis => "vorbis",
         }
     }
 }
@@ -184,11 +237,13 @@ fn parse_details_inner(
     })
 }
 
-#[tracing::instrument(name = "Convert to Mp4", skip(input))]
-pub(crate) async fn to_mp4_bytes(
+#[tracing::instrument(name = "Transcode video", skip(input))]
+pub(crate) async fn trancsocde_bytes(
     input: Bytes,
     input_format: InputFormat,
     permit_audio: bool,
+    video_codec: VideoCodec,
+    audio_codec: Option<AudioCodec>,
 ) -> Result<impl AsyncRead + Unpin, Error> {
     let input_file = crate::tmp_file::tmp_file(Some(input_format.to_ext()));
     let input_file_str = input_file.to_str().ok_or(UploadError::Path)?;
@@ -202,6 +257,9 @@ pub(crate) async fn to_mp4_bytes(
     tmp_one.write_from_bytes(input).await?;
     tmp_one.close().await?;
 
+    let output_format = video_codec.to_output_format();
+    let audio_codec = audio_codec.unwrap_or(output_format.default_audio_codec());
+
     let process = if permit_audio {
         Process::run(
             "ffmpeg",
@@ -213,11 +271,11 @@ pub(crate) async fn to_mp4_bytes(
                 "-vf",
                 "scale=trunc(iw/2)*2:trunc(ih/2)*2",
                 "-c:a",
-                "aac",
+                audio_codec.to_ffmpeg_codec(),
                 "-c:v",
-                "h264",
+                video_codec.to_ffmpeg_codec(),
                 "-f",
-                "mp4",
+                output_format.to_ffmpeg_format(),
                 output_file_str,
             ],
         )?
@@ -233,9 +291,9 @@ pub(crate) async fn to_mp4_bytes(
                 "scale=trunc(iw/2)*2:trunc(ih/2)*2",
                 "-an",
                 "-c:v",
-                "h264",
+                video_codec.to_ffmpeg_codec(),
                 "-f",
-                "mp4",
+                output_format.to_ffmpeg_format(),
                 output_file_str,
             ],
         )?
@@ -281,9 +339,9 @@ pub(crate) async fn thumbnail<S: Store>(
             "-vframes",
             "1",
             "-codec",
-            format.as_codec(),
+            format.as_ffmpeg_codec(),
             "-f",
-            format.as_format(),
+            format.as_ffmpeg_format(),
             output_file_str,
         ],
     )?;

src/generate.rs

@@ -20,6 +20,8 @@ pub(crate) async fn generate<R: FullRepo, S: Store + 'static>(
     alias: Alias,
     thumbnail_path: PathBuf,
     thumbnail_args: Vec<String>,
+    input_format: Option<InputFormat>,
+    thumbnail_format: Option<ThumbnailFormat>,
     hash: R::Bytes,
 ) -> Result<(Details, Bytes), Error> {
     let process_fut = process(
@@ -29,6 +31,8 @@ pub(crate) async fn generate<R: FullRepo, S: Store + 'static>(
         alias,
         thumbnail_path.clone(),
         thumbnail_args,
+        input_format,
+        thumbnail_format,
         hash.clone(),
     );
 
@@ -46,6 +50,8 @@ async fn process<R: FullRepo, S: Store + 'static>(
     alias: Alias,
     thumbnail_path: PathBuf,
     thumbnail_args: Vec<String>,
+    input_format: Option<InputFormat>,
+    thumbnail_format: Option<ThumbnailFormat>,
     hash: R::Bytes,
 ) -> Result<(Details, Bytes), Error> {
     let permit = crate::PROCESS_SEMAPHORE.acquire().await;
@@ -60,8 +66,8 @@ async fn process<R: FullRepo, S: Store + 'static>(
         let reader = crate::ffmpeg::thumbnail(
             store.clone(),
             identifier,
-            InputFormat::Mp4,
-            ThumbnailFormat::Jpeg,
+            input_format.unwrap_or(InputFormat::Mp4),
+            thumbnail_format.unwrap_or(ThumbnailFormat::Jpeg),
         )
         .await?;
         let motion_identifier = store.save_async_read(reader).await?;

src/ingest.rs

@@ -70,11 +70,13 @@ where
     let bytes = aggregate(stream).await?;
 
     tracing::debug!("Validating bytes");
-    let (input_type, validated_reader) = crate::validate::validate_image_bytes(
+    let (input_type, validated_reader) = crate::validate::validate_bytes(
         bytes,
         CONFIG.media.format,
         CONFIG.media.enable_silent_video,
         CONFIG.media.enable_full_video,
+        CONFIG.media.video_codec,
+        CONFIG.media.audio_codec,
         should_validate,
     )
     .await?;

src/main.rs

@@ -48,8 +48,6 @@ mod stream;
 mod tmp_file;
 mod validate;
 
-use crate::magick::ValidInputType;
-
 use self::{
     backgrounded::Backgrounded,
     config::{Configuration, ImageFormat, Operation},
@@ -58,7 +56,7 @@ use self::{
     error::{Error, UploadError},
     ingest::Session,
     init_tracing::init_tracing,
-    magick::details_hint,
+    magick::{details_hint, ValidInputType},
     middleware::{Deadline, Internal},
     queue::queue_generate,
     repo::{
@@ -597,6 +595,7 @@ async fn process<R: FullRepo, S: Store + 'static>(
 
     let path_string = thumbnail_path.to_string_lossy().to_string();
     let hash = repo.hash(&alias).await?;
+
     let identifier_opt = repo
         .variant_identifier::<S::Identifier>(hash.clone(), path_string)
         .await?;
@@ -624,6 +623,8 @@ async fn process<R: FullRepo, S: Store + 'static>(
         return ranged_file_resp(&store, identifier, range, details).await;
     }
 
+    let original_details = ensure_details(&repo, &store, &alias).await?;
+
     let (details, bytes) = generate::generate(
         &repo,
         &store,
@@ -631,6 +632,8 @@ async fn process<R: FullRepo, S: Store + 'static>(
         alias,
         thumbnail_path,
         thumbnail_args,
+        original_details.to_input_format(),
+        None,
         hash,
     )
     .await?;

src/queue/process.rs

@@ -149,6 +149,8 @@ async fn generate<R: FullRepo, S: Store + 'static>(
         return Ok(());
     }
 
+    let original_details = crate::ensure_details(repo, store, &source).await?;
+
     crate::generate::generate(
         repo,
         store,
@@ -156,6 +158,8 @@ async fn generate<R: FullRepo, S: Store + 'static>(
         source,
         process_path,
         process_args,
+        original_details.to_input_format(),
+        None,
         hash,
     )
     .await?;

src/validate.rs

@@ -1,5 +1,5 @@
 use crate::{
-    config::ImageFormat,
+    config::{AudioCodec, ImageFormat, VideoCodec},
     either::Either,
     error::{Error, UploadError},
     ffmpeg::InputFormat,
@@ -36,12 +36,14 @@ impl AsyncRead for UnvalidatedBytes {
     }
 }
 
-#[instrument(name = "Validate image", skip(bytes))]
-pub(crate) async fn validate_image_bytes(
+#[instrument(name = "Validate media", skip(bytes))]
+pub(crate) async fn validate_bytes(
     bytes: Bytes,
     prescribed_format: Option<ImageFormat>,
     enable_silent_video: bool,
     enable_full_video: bool,
+    video_codec: VideoCodec,
+    audio_codec: Option<AudioCodec>,
     validate: bool,
 ) -> Result<(ValidInputType, impl AsyncRead + Unpin), Error> {
     let input_type =
@@ -63,7 +65,14 @@ pub(crate) async fn validate_image_bytes(
             Ok((
                 ValidInputType::Mp4,
                 Either::right(Either::left(
-                    crate::ffmpeg::to_mp4_bytes(bytes, InputFormat::Gif, false).await?,
+                    crate::ffmpeg::trancsocde_bytes(
+                        bytes,
+                        InputFormat::Gif,
+                        false,
+                        video_codec,
+                        audio_codec,
+                    )
+                    .await?,
                 )),
             ))
         }
@@ -74,7 +83,14 @@ pub(crate) async fn validate_image_bytes(
             Ok((
                 ValidInputType::Mp4,
                 Either::right(Either::left(
-                    crate::ffmpeg::to_mp4_bytes(bytes, InputFormat::Mp4, enable_full_video).await?,
+                    crate::ffmpeg::trancsocde_bytes(
+                        bytes,
+                        InputFormat::Mp4,
+                        enable_full_video,
+                        video_codec,
+                        audio_codec,
+                    )
+                    .await?,
                 )),
             ))
         }
@@ -85,7 +101,13 @@ pub(crate) async fn validate_image_bytes(
             Ok((
                 ValidInputType::Mp4,
                 Either::right(Either::left(
-                    crate::ffmpeg::to_mp4_bytes(bytes, InputFormat::Webm, enable_full_video)
+                    crate::ffmpeg::trancsocde_bytes(
+                        bytes,
+                        InputFormat::Webm,
+                        enable_full_video,
+                        video_codec,
+                        audio_codec,
+                    )
                     .await?,
                 )),
             ))