It compiles and runs, but doesn't work

This commit is contained in:
asonix 2023-07-13 17:42:21 -05:00
parent 27451971a6
commit 2c22f2ee3a
18 changed files with 881 additions and 434 deletions

View File

@ -18,14 +18,7 @@ targets = "info"
path = "/mnt" path = "/mnt"
[media] [media]
max_width = 10000
max_height = 10000
max_area = 40000000
max_file_size = 40 max_file_size = 40
max_frame_count = 900
enable_silent_video = true
enable_full_video = false
video_codec = "vp9"
filters = [ filters = [
"blur", "blur",
"crop", "crop",
@ -33,14 +26,30 @@ filters = [
"resize", "resize",
"thumbnail", "thumbnail",
] ]
skip_validate_imports = false
[media.gif] [media.image]
max_width = 128 max_width = 10000
max_height = 128 max_height = 10000
max_area = 16384 max_area = 40000000
max_file_size = 40
[media.animation]
max_width = 256
max_height = 256
max_area = 65536
max_file_size = 40
max_frame_count = 100 max_frame_count = 100
[media.video]
enable = true
allow_audio = false
max_width = 3840
max_height = 3840
max_area = 8294400
max_file_size = 40
max_frame_count = 900
video_codec = "vp9"
[repo] [repo]
type = "sled" type = "sled"
path = "/mnt/sled-repo" path = "/mnt/sled-repo"

View File

@ -128,6 +128,11 @@ path = '/mnt'
## Media Processing Configuration ## Media Processing Configuration
[media] [media]
## Optional: max file size (in Megabytes)
# environment variable: PICTRS__MEDIA__MAX_FILE_SIZE
# default: 40
max_file_size = 40
## Optional: preprocessing steps for uploaded images ## Optional: preprocessing steps for uploaded images
# environment variable: PICTRS__MEDIA__PREPROCESS_STEPS # environment variable: PICTRS__MEDIA__PREPROCESS_STEPS
# default: empty # default: empty
@ -135,70 +140,35 @@ path = '/mnt'
# This configuration is the same format as the process endpoint's query arguments # This configuration is the same format as the process endpoint's query arguments
preprocess_steps = 'crop=16x9&resize=1200&blur=0.2' preprocess_steps = 'crop=16x9&resize=1200&blur=0.2'
## Optional: max media width (in pixels)
# environment variable: PICTRS__MEDIA__MAX_WIDTH
# default: 10,000
max_width = 10000
## Optional: max media height (in pixels)
# environment variable: PICTRS__MEDIA__MAX_HEIGHT
# default: 10,000
max_height = 10000
## Optional: max media area (in pixels)
# environment variable: PICTRS__MEDIA__MAX_AREA
# default: 40,000,000
max_area = 40000000
## Optional: max file size (in Megabytes)
# environment variable: PICTRS__MEDIA__MAX_FILE_SIZE
# default: 40
max_file_size = 40
## Optional: max frame count
# environment variable: PICTRS__MEDIA__MAX_FRAME_COUNT
# default: # 900
max_frame_count = 900
## Optional: enable GIF, MP4, and WEBM uploads (without sound)
# environment variable: PICTRS__MEDIA__ENABLE_SILENT_VIDEO
# default: true
#
# Set this to false to serve static images only
enable_silent_video = true
## Optional: enable MP4, and WEBM uploads (with sound) and GIF (without sound)
# environment variable: PICTRS__MEDIA__ENABLE_FULL_VIDEO
# default: false
enable_full_video = false
## Optional: set the default video codec
# environment variable: PICTRS__MEDIA__VIDEO_CODEC
# default: vp9
#
# available options: av1, h264, h265, vp8, vp9
# this setting does nothing if video is not enabled
video_codec = "vp9"
## Optional: set the default audio codec
# environment variable: PICTRS__MEDIA__AUDIO_CODEC
# default: empty
#
# available options: aac, opus, vorbis
# The audio codec is automatically selected based on video codec, but can be overriden
# av1, vp8, and vp9 map to opus
# h264 and h265 map to aac
# vorbis is not default for any codec
# this setting does nothing if full video is not enabled
audio_codec = "aac"
## Optional: set allowed filters for image processing ## Optional: set allowed filters for image processing
# environment variable: PICTRS__MEDIA__FILTERS # environment variable: PICTRS__MEDIA__FILTERS
# default: ['blur', 'crop', 'identity', 'resize', 'thumbnail'] # default: ['blur', 'crop', 'identity', 'resize', 'thumbnail']
filters = ['blur', 'crop', 'identity', 'resize', 'thumbnail'] filters = ['blur', 'crop', 'identity', 'resize', 'thumbnail']
## Optional: set file type for all uploads
# environment variable: PICTRS__MEDIA__FORMAT [media.image]
## Optional: max media width (in pixels)
# environment variable: PICTRS__MEDIA__IMAGE__MAX_WIDTH
# default: 10,000
max_width = 10000
## Optional: max media height (in pixels)
# environment variable: PICTRS__MEDIA__IMAGE__MAX_HEIGHT
# default: 10,000
max_height = 10000
## Optional: max media area (in pixels)
# environment variable: PICTRS__MEDIA__IMAGE__MAX_AREA
# default: 40,000,000
max_area = 40000000
## Optional: max file size (in Megabytes)
# environment variable: PICTRS__MEDIA__IMAGE__MAX_FILE_SIZE
# default: 40
max_file_size = 40
## Optional: set file type for all images
# environment variable: PICTRS__MEDIA__IMAGE__FORMAT
# default: empty # default: empty
# #
# available options: avif, png, jpeg, jxl, webp # available options: avif, png, jpeg, jxl, webp
@ -207,49 +177,126 @@ filters = ['blur', 'crop', 'identity', 'resize', 'thumbnail']
# are stored in their original file type. # are stored in their original file type.
format = "webp" format = "webp"
## Optional: whether to validate images uploaded through the `import` endpoint
# environment variable: PICTRS__MEDIA__SKIP_VALIDATE_IMPORTS
# default: false
#
# Set this to true if you want to avoid processing imported media
skip_validate_imports = false
## Gif configuration [media.animation]
## Optional: max animation width (in pixels)
# environment variable: PICTRS__MEDIA__ANIMATION__MAX_WIDTH
# default: 256
# #
# Making any of these bounds 0 will disable gif uploads # If an animation exceeds this value, it may be converted to a silent video
[media.gif] max_width = 256
# Optional: Maximum width in pixels for uploaded gifs
# environment variable: PICTRS__MEDIA__GIF__MAX_WIDTH
# default: 128
#
# If a gif does not fit within this bound, it will either be transcoded to a video or rejected,
# depending on whether video uploads are enabled
max_width = 128
# Optional: Maximum height in pixels for uploaded gifs ## Optional: max animation height (in pixels)
# environment variable: PICTRS__MEDIA__GIF__MAX_HEIGHT # environment variable: PICTRS__MEDIA__ANIMATION__MAX_HEIGHT
# default: 128 # default: 256
# #
# If a gif does not fit within this bound, it will either be transcoded to a video or rejected, # If an animation exceeds this value, it may be converted to a silent video
# depending on whether video uploads are enabled max_height = 256
max_height = 128
# Optional: Maximum area in pixels for uploaded gifs ## Optional: max animation area (in pixels)
# environment variable: PICTRS__MEDIA__GIF__MAX_AREA # environment variable: PICTRS__MEDIA__ANIMATION__MAX_AREA
# default: 16384 (128 * 128) # default: 65,526
# #
# If a gif does not fit within this bound, it will either be transcoded to a video or rejected, # If an animation exceeds this value, it may be converted to a silent video
# depending on whether video uploads are enabled max_area = 65536
max_area = 16384
# Optional: Maximum number of frames permitted in uploaded gifs ## Optional: max animation size (in Megabytes)
# environment variable: PICTRS__MEDIA__GIF__MAX_FRAME_COUNT # environment variable: PICTRS__MEDIA__ANIMATION__MAX_FILE_SIZE
# default: 40
#
# If an animation exceeds this value, it may be converted to a silent video
max_file_size = 40
## Optional: max frame count
# environment variable: PICTRS__MEDIA__ANIMATION__MAX_FRAME_COUNT
# default: 100 # default: 100
# #
# If a gif does not fit within this bound, it will either be transcoded to a video or rejected, # If an animation exceeds this value, it may be converted to a silent video
# depending on whether video uploads are enabled
max_frame_count = 100 max_frame_count = 100
## Optional: set file type for all animations
# environment variable: PICTRS__MEDIA__ANIMATION__FORMAT
# default: empty
#
# available options: apng, avif, gif, webp
# When set, all uploaded still images will be converted to this file type. For balancing quality vs
# file size vs browser support, 'avif', 'jxl', and 'webp' should be considered. By default, images
# are stored in their original file type.
format = "webp"
[media.video]
## Optional: enable MP4 and WEBM uploads (without sound)
# environment variable: PICTRS__MEDIA__VIDEO__ENABLE
# default: true
#
# Set this to false to serve static images only
enable = true
## Optional: enable Sound for MP4 and WEBM uploads
# environment variable: PICTRS__MEDIA__VIDEO__ALLOW_AUDIO
# default: false
#
# this setting does nothing if video is not enabled
allow_audio = false
## Optional: max video width (in pixels)
# environment variable: PICTRS__MEDIA__VIDEO__MAX_WIDTH
# default: 3,840
#
# this setting does nothing if video is not enabled
max_width = 3840
## Optional: max video height (in pixels)
# environment variable: PICTRS__MEDIA__VIDEO__MAX_HEIGHT
# default: 3,840
#
# this setting does nothing if video is not enabled
max_height = 3840
## Optional: max video area (in pixels)
# environment variable: PICTRS__MEDIA__VIDEO__MAX_AREA
# default: 8,294,400
#
# this setting does nothing if video is not enabled
max_area = 8294400
## Optional: max video size (in Megabytes)
# environment variable: PICTRS__MEDIA__VIDEO__MAX_FILE_SIZE
# default: 40
#
# this setting does nothing if video is not enabled
max_file_size = 40
## Optional: max frame count
# environment variable: PICTRS__MEDIA__VIDEO__MAX_FRAME_COUNT
# default: 900
#
# this setting does nothing if video is not enabled
max_frame_count = 900
## Optional: set the default video codec
# environment variable: PICTRS__MEDIA__VIDEO__VIDEO_CODEC
# default: vp9
#
# available options: av1, h264, h265, vp8, vp9
# this setting does nothing if video is not enabled
video_codec = "vp9"
## Optional: set the default audio codec
# environment variable: PICTRS__MEDIA__VIDEO__AUDIO_CODEC
# default: empty
#
# available options: aac, opus, vorbis
# The audio codec is automatically selected based on video codec, but can be overriden to `vorbis`
# for webm uploads
# automatic mappings:
# - av1, vp8, and vp9 map to opus
# - h264 and h265 map to aac
# - vorbis is not default for any codec
# this setting does nothing if full video is not enabled
audio_codec = "opus"
## Database configuration ## Database configuration
[repo] [repo]

View File

@ -12,7 +12,8 @@ use defaults::Defaults;
pub(crate) use commandline::Operation; pub(crate) use commandline::Operation;
pub(crate) use file::{ pub(crate) use file::{
ConfigFile as Configuration, ObjectStorage, OpenTelemetry, Repo, Sled, Store, Tracing, Animation, ConfigFile as Configuration, Image, Media, ObjectStorage, OpenTelemetry, Repo, Sled,
Store, Tracing, Video,
}; };
pub(crate) use primitives::{Filesystem, LogFormat}; pub(crate) use primitives::{Filesystem, LogFormat};

View File

@ -1,6 +1,6 @@
use crate::{ use crate::{
config::primitives::{AudioCodec, LogFormat, Targets, VideoCodec}, config::primitives::{LogFormat, Targets},
formats::ImageFormat, formats::{AnimationFormat, AudioCodec, ImageFormat, VideoCodec},
serde_str::Serde, serde_str::Serde,
}; };
use clap::{Parser, Subcommand}; use clap::{Parser, Subcommand};
@ -48,21 +48,28 @@ impl Args {
worker_id, worker_id,
client_pool_size, client_pool_size,
media_preprocess_steps, media_preprocess_steps,
media_skip_validate_imports,
media_max_width,
media_max_height,
media_max_area,
media_max_file_size, media_max_file_size,
media_max_frame_count, media_image_max_width,
media_gif_max_width, media_image_max_height,
media_gif_max_height, media_image_max_area,
media_gif_max_area, media_image_max_file_size,
media_enable_silent_video, media_image_format,
media_enable_full_video, media_animation_max_width,
media_animation_max_height,
media_animation_max_area,
media_animation_max_file_size,
media_animation_max_frame_count,
media_animation_format,
media_video_enable,
media_video_allow_audio,
media_video_max_width,
media_video_max_height,
media_video_max_area,
media_video_max_file_size,
media_video_max_frame_count,
media_video_codec, media_video_codec,
media_audio_codec, media_video_audio_codec,
media_filters, media_filters,
media_format,
store, store,
}) => { }) => {
let server = Server { let server = Server {
@ -71,33 +78,43 @@ impl Args {
worker_id, worker_id,
client_pool_size, client_pool_size,
}; };
let gif = if media_gif_max_width.is_none()
&& media_gif_max_height.is_none() let image = Image {
&& media_gif_max_area.is_none() max_file_size: media_image_max_file_size,
{ max_width: media_image_max_width,
None max_height: media_image_max_height,
} else { max_area: media_image_max_area,
Some(Gif { format: media_image_format,
max_width: media_gif_max_width,
max_height: media_gif_max_height,
max_area: media_gif_max_area,
})
}; };
let media = Media {
preprocess_steps: media_preprocess_steps, let animation = Animation {
skip_validate_imports: media_skip_validate_imports, max_file_size: media_animation_max_file_size,
max_width: media_max_width, max_width: media_animation_max_width,
max_height: media_max_height, max_height: media_animation_max_height,
max_area: media_max_area, max_area: media_animation_max_area,
max_file_size: media_max_file_size, max_frame_count: media_animation_max_frame_count,
max_frame_count: media_max_frame_count, format: media_animation_format,
gif, };
enable_silent_video: media_enable_silent_video,
enable_full_video: media_enable_full_video, let video = Video {
enable: media_video_enable,
allow_audio: media_video_allow_audio,
max_file_size: media_video_max_file_size,
max_width: media_video_max_width,
max_height: media_video_max_height,
max_area: media_video_max_area,
max_frame_count: media_video_max_frame_count,
video_codec: media_video_codec, video_codec: media_video_codec,
audio_codec: media_audio_codec, audio_codec: media_video_audio_codec,
};
let media = Media {
max_file_size: media_max_file_size,
preprocess_steps: media_preprocess_steps,
filters: media_filters, filters: media_filters,
format: media_format, image: image.set(),
animation: animation.set(),
video: video.set(),
}; };
let operation = Operation::Run; let operation = Operation::Run;
@ -335,45 +352,126 @@ struct OldDb {
#[derive(Debug, Default, serde::Serialize)] #[derive(Debug, Default, serde::Serialize)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
struct Media { struct Media {
#[serde(skip_serializing_if = "Option::is_none")]
preprocess_steps: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
max_width: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
max_height: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
max_area: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
max_file_size: Option<usize>, max_file_size: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
max_frame_count: Option<usize>, preprocess_steps: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
gif: Option<Gif>,
#[serde(skip_serializing_if = "Option::is_none")]
enable_silent_video: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
enable_full_video: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
video_codec: Option<VideoCodec>,
#[serde(skip_serializing_if = "Option::is_none")]
audio_codec: Option<AudioCodec>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
filters: Option<Vec<String>>, filters: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
format: Option<ImageFormat>, image: Option<Image>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
skip_validate_imports: Option<bool>, animation: Option<Animation>,
#[serde(skip_serializing_if = "Option::is_none")]
video: Option<Video>,
} }
#[derive(Debug, Default, serde::Serialize)] #[derive(Debug, Default, serde::Serialize)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
struct Gif { struct Image {
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
max_width: Option<usize>, max_width: Option<u16>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
max_height: Option<usize>, max_height: Option<u16>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
max_area: Option<usize>, max_area: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
max_file_size: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
format: Option<ImageFormat>,
}
impl Image {
fn set(self) -> Option<Self> {
let any_set = self.max_width.is_some()
|| self.max_height.is_some()
|| self.max_area.is_some()
|| self.max_file_size.is_some()
|| self.format.is_some();
if any_set {
Some(self)
} else {
None
}
}
}
#[derive(Debug, Default, serde::Serialize)]
#[serde(rename_all = "snake_case")]
struct Animation {
#[serde(skip_serializing_if = "Option::is_none")]
max_width: Option<u16>,
#[serde(skip_serializing_if = "Option::is_none")]
max_height: Option<u16>,
#[serde(skip_serializing_if = "Option::is_none")]
max_area: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
max_frame_count: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
max_file_size: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
format: Option<AnimationFormat>,
}
impl Animation {
fn set(self) -> Option<Self> {
let any_set = self.max_width.is_some()
|| self.max_height.is_some()
|| self.max_area.is_some()
|| self.max_frame_count.is_some()
|| self.max_file_size.is_some()
|| self.format.is_some();
if any_set {
Some(self)
} else {
None
}
}
}
#[derive(Debug, Default, serde::Serialize)]
#[serde(rename_all = "snake_case")]
struct Video {
#[serde(skip_serializing_if = "Option::is_none")]
enable: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
allow_audio: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
max_width: Option<u16>,
#[serde(skip_serializing_if = "Option::is_none")]
max_height: Option<u16>,
#[serde(skip_serializing_if = "Option::is_none")]
max_area: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
max_frame_count: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
max_file_size: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
video_codec: Option<VideoCodec>,
#[serde(skip_serializing_if = "Option::is_none")]
audio_codec: Option<AudioCodec>,
}
impl Video {
fn set(self) -> Option<Self> {
let any_set = self.enable.is_some()
|| self.allow_audio.is_some()
|| self.max_width.is_some()
|| self.max_height.is_some()
|| self.max_area.is_some()
|| self.max_frame_count.is_some()
|| self.max_file_size.is_some()
|| self.video_codec.is_some()
|| self.audio_codec.is_some();
if any_set {
Some(self)
} else {
None
}
}
} }
/// Run the pict-rs application /// Run the pict-rs application
@ -457,60 +555,75 @@ struct Run {
#[arg(long)] #[arg(long)]
media_preprocess_steps: Option<String>, media_preprocess_steps: Option<String>,
/// Whether to validate media on the "import" endpoint /// Which media filters should be enabled on the `process` endpoint
#[arg(long)] #[arg(long)]
media_skip_validate_imports: Option<bool>, media_filters: Option<Vec<String>>,
/// The maximum width, in pixels, for uploaded media
#[arg(long)] /// The maximum size, in megabytes, for all uploaded media
media_max_width: Option<usize>,
/// The maximum height, in pixels, for uploaded media
#[arg(long)]
media_max_height: Option<usize>,
/// The maximum area, in pixels, for uploaded media
#[arg(long)]
media_max_area: Option<usize>,
/// The maximum size, in megabytes, for uploaded media
#[arg(long)] #[arg(long)]
media_max_file_size: Option<usize>, media_max_file_size: Option<usize>,
/// The maximum number of frames allowed for uploaded GIF and MP4s.
/// The maximum width, in pixels, for uploaded images
#[arg(long)] #[arg(long)]
media_max_frame_count: Option<usize>, media_image_max_width: Option<u16>,
/// Maximum width allowed for gif uploads. /// The maximum height, in pixels, for uploaded images
///
/// If an upload exceeds this value, it will be transcoded to a video format or rejected,
/// depending on whether video uploads are enabled.
#[arg(long)] #[arg(long)]
media_gif_max_width: Option<usize>, media_image_max_height: Option<u16>,
/// Maximum height allowed for gif uploads /// The maximum area, in pixels, for uploaded images
///
/// If an upload exceeds this value, it will be transcoded to a video format or rejected,
/// depending on whether video uploads are enabled.
#[arg(long)] #[arg(long)]
media_gif_max_height: Option<usize>, media_image_max_area: Option<u32>,
/// Maximum area allowed for gif uploads /// The maximum size, in megabytes, for uploaded images
///
/// If an upload exceeds this value, it will be transcoded to a video format or rejected,
/// depending on whether video uploads are enabled.
#[arg(long)] #[arg(long)]
media_gif_max_area: Option<usize>, media_image_max_file_size: Option<usize>,
/// Whether to enable GIF and silent video uploads /// Enforce a specific format for uploaded images
#[arg(long)] #[arg(long)]
media_enable_silent_video: Option<bool>, media_image_format: Option<ImageFormat>,
/// Whether to enable full video uploads
/// The maximum width, in pixels, for uploaded animations
#[arg(long)] #[arg(long)]
media_enable_full_video: Option<bool>, media_animation_max_width: Option<u16>,
/// The maximum height, in pixels, for uploaded animations
#[arg(long)]
media_animation_max_height: Option<u16>,
/// The maximum area, in pixels, for uploaded animations
#[arg(long)]
media_animation_max_area: Option<u32>,
/// The maximum number of frames allowed for uploaded animations
#[arg(long)]
media_animation_max_frame_count: Option<u32>,
/// The maximum size, in megabytes, for uploaded animations
#[arg(long)]
media_animation_max_file_size: Option<usize>,
/// Enforce a specific format for uploaded animations
#[arg(long)]
media_animation_format: Option<AnimationFormat>,
/// Whether to enable video uploads
#[arg(long)]
media_video_enable: Option<bool>,
/// Whether to enable audio in video uploads
media_video_allow_audio: Option<bool>,
/// The maximum width, in pixels, for uploaded videos
#[arg(long)]
media_video_max_width: Option<u16>,
/// The maximum height, in pixels, for uploaded videos
#[arg(long)]
media_video_max_height: Option<u16>,
/// The maximum area, in pixels, for uploaded videos
#[arg(long)]
media_video_max_area: Option<u32>,
/// The maximum number of frames allowed for uploaded videos
#[arg(long)]
media_video_max_frame_count: Option<u32>,
/// The maximum size, in megabytes, for uploaded videos
#[arg(long)]
media_video_max_file_size: Option<usize>,
/// Enforce a specific video codec for uploaded videos /// Enforce a specific video codec for uploaded videos
#[arg(long)] #[arg(long)]
media_video_codec: Option<VideoCodec>, media_video_codec: Option<VideoCodec>,
/// Enforce a specific audio codec for uploaded videos /// Enforce a specific audio codec for uploaded videos
#[arg(long)] #[arg(long)]
media_audio_codec: Option<AudioCodec>, media_video_audio_codec: Option<AudioCodec>,
/// Which media filters should be enabled on the `process` endpoint
#[arg(long)]
media_filters: Option<Vec<String>>,
/// Enforce uploaded media is transcoded to the provided format
#[arg(long)]
media_format: Option<ImageFormat>,
#[command(subcommand)] #[command(subcommand)]
store: Option<RunStore>, store: Option<RunStore>,

View File

@ -1,5 +1,6 @@
use crate::{ use crate::{
config::primitives::{LogFormat, Targets, VideoCodec}, config::primitives::{LogFormat, Targets},
formats::VideoCodec,
serde_str::Serde, serde_str::Serde,
}; };
use std::{net::SocketAddr, path::PathBuf}; use std::{net::SocketAddr, path::PathBuf};
@ -62,26 +63,43 @@ struct OldDbDefaults {
#[derive(Clone, Debug, serde::Serialize)] #[derive(Clone, Debug, serde::Serialize)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
struct MediaDefaults { struct MediaDefaults {
max_width: usize,
max_height: usize,
max_area: usize,
max_file_size: usize, max_file_size: usize,
max_frame_count: usize,
gif: GifDefaults,
enable_silent_video: bool,
enable_full_video: bool,
video_codec: VideoCodec,
filters: Vec<String>, filters: Vec<String>,
skip_validate_imports: bool, image: ImageDefaults,
animation: AnimationDefaults,
video: VideoDefaults,
} }
#[derive(Clone, Debug, serde::Serialize)] #[derive(Clone, Debug, serde::Serialize)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
struct GifDefaults { struct ImageDefaults {
max_height: usize, max_width: u16,
max_width: usize, max_height: u16,
max_area: usize, max_area: u32,
max_frame_count: usize, max_file_size: usize,
}
#[derive(Clone, Debug, serde::Serialize)]
#[serde(rename_all = "snake_case")]
struct AnimationDefaults {
max_width: u16,
max_height: u16,
max_area: u32,
max_frame_count: u32,
max_file_size: usize,
}
#[derive(Clone, Debug, serde::Serialize)]
#[serde(rename_all = "snake_case")]
struct VideoDefaults {
enable: bool,
allow_audio: bool,
max_height: u16,
max_width: u16,
max_area: u32,
max_frame_count: u32,
max_file_size: usize,
video_codec: VideoCodec,
} }
#[derive(Clone, Debug, serde::Serialize)] #[derive(Clone, Debug, serde::Serialize)]
@ -175,15 +193,7 @@ impl Default for OldDbDefaults {
impl Default for MediaDefaults { impl Default for MediaDefaults {
fn default() -> Self { fn default() -> Self {
MediaDefaults { MediaDefaults {
max_width: 10_000,
max_height: 10_000,
max_area: 40_000_000,
max_file_size: 40, max_file_size: 40,
max_frame_count: 900,
gif: Default::default(),
enable_silent_video: true,
enable_full_video: false,
video_codec: VideoCodec::Vp9,
filters: vec![ filters: vec![
"blur".into(), "blur".into(),
"crop".into(), "crop".into(),
@ -191,18 +201,47 @@ impl Default for MediaDefaults {
"resize".into(), "resize".into(),
"thumbnail".into(), "thumbnail".into(),
], ],
skip_validate_imports: false, image: Default::default(),
animation: Default::default(),
video: Default::default(),
} }
} }
} }
impl Default for GifDefaults { impl Default for ImageDefaults {
fn default() -> Self { fn default() -> Self {
GifDefaults { ImageDefaults {
max_height: 128, max_width: 10_000,
max_width: 128, max_height: 10_000,
max_area: 16384, max_area: 40_000_000,
max_file_size: 40,
}
}
}
impl Default for AnimationDefaults {
fn default() -> Self {
AnimationDefaults {
max_height: 256,
max_width: 256,
max_area: 65_536,
max_frame_count: 100, max_frame_count: 100,
max_file_size: 40,
}
}
}
impl Default for VideoDefaults {
fn default() -> Self {
VideoDefaults {
enable: true,
allow_audio: false,
max_height: 3_840,
max_width: 3_840,
max_area: 8_294_400,
max_frame_count: 900,
max_file_size: 40,
video_codec: VideoCodec::Vp9,
} }
} }
} }

View File

@ -1,6 +1,6 @@
use crate::{ use crate::{
config::primitives::{AudioCodec, Filesystem, LogFormat, Targets, VideoCodec}, config::primitives::{Filesystem, LogFormat, Targets},
formats::ImageFormat, formats::{AnimationFormat, AudioCodec, ImageFormat, VideoCodec},
serde_str::Serde, serde_str::Serde,
}; };
use once_cell::sync::OnceCell; use once_cell::sync::OnceCell;
@ -144,47 +144,70 @@ pub(crate) struct OldDb {
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
pub(crate) struct Media { pub(crate) struct Media {
pub(crate) max_file_size: usize,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub(crate) preprocess_steps: Option<String>, pub(crate) preprocess_steps: Option<String>,
pub(crate) max_width: usize, pub(crate) filters: BTreeSet<String>,
pub(crate) max_height: usize, pub(crate) image: Image,
pub(crate) max_area: usize, pub(crate) animation: Animation,
pub(crate) video: Video,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub(crate) struct Image {
pub(crate) max_width: u16,
pub(crate) max_height: u16,
pub(crate) max_area: u32,
pub(crate) max_file_size: usize, pub(crate) max_file_size: usize,
pub(crate) max_frame_count: usize, #[serde(skip_serializing_if = "Option::is_none")]
pub(crate) format: Option<ImageFormat>,
}
pub(crate) gif: Gif, #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub(crate) struct Animation {
pub(crate) max_width: u16,
pub(crate) enable_silent_video: bool, pub(crate) max_height: u16,
pub(crate) enable_full_video: bool, pub(crate) max_area: u32,
pub(crate) max_file_size: usize,
pub(crate) max_frame_count: u32,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) format: Option<AnimationFormat>,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub(crate) struct Video {
pub(crate) enable: bool,
pub(crate) allow_audio: bool,
pub(crate) max_width: u16,
pub(crate) max_height: u16,
pub(crate) max_area: u32,
pub(crate) max_file_size: usize,
pub(crate) max_frame_count: u32,
pub(crate) video_codec: VideoCodec, pub(crate) video_codec: VideoCodec,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
pub(crate) audio_codec: Option<AudioCodec>, pub(crate) audio_codec: Option<AudioCodec>,
pub(crate) filters: BTreeSet<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) format: Option<ImageFormat>,
pub(crate) skip_validate_imports: bool,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub(crate) struct Gif {
pub(crate) max_width: usize,
pub(crate) max_height: usize,
pub(crate) max_area: usize,
pub(crate) max_frame_count: usize,
} }
impl Media { impl Media {

View File

@ -24,48 +24,6 @@ pub(crate) enum LogFormat {
Pretty, Pretty,
} }
#[derive(
Clone,
Copy,
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Hash,
serde::Deserialize,
serde::Serialize,
ValueEnum,
)]
#[serde(rename_all = "snake_case")]
pub(crate) enum VideoCodec {
H264,
H265,
Av1,
Vp8,
Vp9,
}
#[derive(
Clone,
Copy,
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Hash,
serde::Deserialize,
serde::Serialize,
ValueEnum,
)]
#[serde(rename_all = "snake_case")]
pub(crate) enum AudioCodec {
Aac,
Opus,
Vorbis,
}
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub(crate) struct Targets { pub(crate) struct Targets {
pub(crate) targets: tracing_subscriber::filter::Targets, pub(crate) targets: tracing_subscriber::filter::Targets,

View File

@ -6,7 +6,7 @@ use std::{collections::HashSet, sync::OnceLock};
use crate::{ use crate::{
ffmpeg::FfMpegError, ffmpeg::FfMpegError,
formats::{ formats::{
AnimationFormat, AnimationInput, ImageFormat, ImageInput, InputFile, InternalFormat, AnimationFormat, ImageFormat, ImageInput, InputFile, InternalFormat,
InternalVideoFormat, VideoFormat, InternalVideoFormat, VideoFormat,
}, },
process::Process, process::Process,
@ -169,7 +169,7 @@ where
frames, frames,
})), })),
InternalFormat::Animation(format) => Ok(Some(Discovery { InternalFormat::Animation(format) => Ok(Some(Discovery {
input: InputFile::Animation(AnimationInput { format }), input: InputFile::Animation(format),
width, width,
height, height,
frames, frames,

View File

@ -6,7 +6,7 @@ use futures_util::Stream;
use tokio::io::AsyncReadExt; use tokio::io::AsyncReadExt;
use crate::{ use crate::{
formats::{AnimationFormat, AnimationInput, ImageFormat, ImageInput, InputFile, VideoFormat}, formats::{AnimationFormat, ImageFormat, ImageInput, InputFile, VideoFormat},
magick::MagickError, magick::MagickError,
process::Process, process::Process,
}; };
@ -78,9 +78,7 @@ pub(super) async fn confirm_bytes(
match discovery { match discovery {
Some(Discovery { Some(Discovery {
input: input:
InputFile::Animation(AnimationInput { InputFile::Animation( AnimationFormat::Avif,),
format: AnimationFormat::Avif,
}),
width, width,
height, height,
.. ..
@ -94,9 +92,7 @@ pub(super) async fn confirm_bytes(
.await?; .await?;
return Ok(Discovery { return Ok(Discovery {
input: InputFile::Animation(AnimationInput { input: InputFile::Animation( AnimationFormat::Avif,),
format: AnimationFormat::Avif,
}),
width, width,
height, height,
frames: Some(frames), frames: Some(frames),
@ -104,9 +100,7 @@ pub(super) async fn confirm_bytes(
} }
Some(Discovery { Some(Discovery {
input: input:
InputFile::Animation(AnimationInput { InputFile::Animation( AnimationFormat::Webp,),
format: AnimationFormat::Webp,
}),
.. ..
}) => { }) => {
// continue // continue
@ -265,9 +259,7 @@ fn parse_discovery(output: Vec<MagickDiscovery>) -> Result<Discovery, MagickErro
"AVIF" => { "AVIF" => {
if frames > 1 { if frames > 1 {
Ok(Discovery { Ok(Discovery {
input: InputFile::Animation(AnimationInput { input: InputFile::Animation( AnimationFormat::Avif,),
format: AnimationFormat::Avif,
}),
width, width,
height, height,
frames: Some(frames), frames: Some(frames),
@ -285,17 +277,13 @@ fn parse_discovery(output: Vec<MagickDiscovery>) -> Result<Discovery, MagickErro
} }
} }
"APNG" => Ok(Discovery { "APNG" => Ok(Discovery {
input: InputFile::Animation(AnimationInput { input: InputFile::Animation( AnimationFormat::Apng,),
format: AnimationFormat::Apng,
}),
width, width,
height, height,
frames: Some(frames), frames: Some(frames),
}), }),
"GIF" => Ok(Discovery { "GIF" => Ok(Discovery {
input: InputFile::Animation(AnimationInput { input: InputFile::Animation( AnimationFormat::Gif,),
format: AnimationFormat::Gif,
}),
width, width,
height, height,
frames: Some(frames), frames: Some(frames),
@ -336,9 +324,7 @@ fn parse_discovery(output: Vec<MagickDiscovery>) -> Result<Discovery, MagickErro
"WEBP" => { "WEBP" => {
if frames > 1 { if frames > 1 {
Ok(Discovery { Ok(Discovery {
input: InputFile::Animation(AnimationInput { input: InputFile::Animation( AnimationFormat::Webp,),
format: AnimationFormat::Webp,
}),
width, width,
height, height,
frames: Some(frames), frames: Some(frames),

View File

@ -57,6 +57,9 @@ pub(crate) enum UploadError {
#[error("Error interacting with filesystem")] #[error("Error interacting with filesystem")]
Io(#[from] std::io::Error), Io(#[from] std::io::Error),
#[error("Error validating upload")]
Validation(#[from] crate::validate::ValidationError),
#[error("Error generating path")] #[error("Error generating path")]
PathGenerator(#[from] storage_path_generator::PathError), PathGenerator(#[from] storage_path_generator::PathError),
@ -166,6 +169,7 @@ impl ResponseError for Error {
crate::repo::RepoError::AlreadyClaimed, crate::repo::RepoError::AlreadyClaimed,
)) ))
| UploadError::Repo(crate::repo::RepoError::AlreadyClaimed) | UploadError::Repo(crate::repo::RepoError::AlreadyClaimed)
| UploadError::Validation(_)
| UploadError::UnsupportedProcessExtension, | UploadError::UnsupportedProcessExtension,
) => StatusCode::BAD_REQUEST, ) => StatusCode::BAD_REQUEST,
Some(UploadError::Magick(e)) if e.is_client_error() => StatusCode::BAD_REQUEST, Some(UploadError::Magick(e)) if e.is_client_error() => StatusCode::BAD_REQUEST,

View File

@ -5,22 +5,21 @@ mod video;
use std::str::FromStr; use std::str::FromStr;
pub(crate) use animation::{AnimationFormat, AnimationInput, AnimationOutput}; pub(crate) use animation::{AnimationFormat, AnimationOutput};
pub(crate) use image::{ImageFormat, ImageInput, ImageOutput}; pub(crate) use image::{ImageFormat, ImageInput, ImageOutput};
pub(crate) use video::{InternalVideoFormat, OutputVideoFormat, VideoFormat}; pub(crate) use video::{InternalVideoFormat, OutputVideoFormat, VideoFormat, VideoCodec, AudioCodec};
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub(crate) struct PrescribedFormats { pub(crate) struct Validations<'a> {
pub(crate) image: Option<ImageFormat>, pub(crate) image: &'a crate::config::Image,
pub(crate) animation: Option<AnimationFormat>, pub(crate) animation: &'a crate::config::Animation,
pub(crate) video: Option<OutputVideoFormat>, pub(crate) video: &'a crate::config::Video,
pub(crate) allow_audio: bool,
} }
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) enum InputFile { pub(crate) enum InputFile {
Image(ImageInput), Image(ImageInput),
Animation(AnimationInput), Animation(AnimationFormat),
Video(VideoFormat), Video(VideoFormat),
} }
@ -61,7 +60,7 @@ impl InputFile {
pub(crate) const fn internal_format(&self) -> InternalFormat { pub(crate) const fn internal_format(&self) -> InternalFormat {
match self { match self {
Self::Image(ImageInput { format, .. }) => InternalFormat::Image(*format), Self::Image(ImageInput { format, .. }) => InternalFormat::Image(*format),
Self::Animation(AnimationInput { format }) => InternalFormat::Animation(*format), Self::Animation(format) => InternalFormat::Animation(*format),
Self::Video(format) => InternalFormat::Video(format.internal_format()), Self::Video(format) => InternalFormat::Video(format.internal_format()),
} }
} }

View File

@ -1,5 +1,15 @@
#[derive( #[derive(
Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Deserialize, serde::Serialize, Clone,
Copy,
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Hash,
serde::Deserialize,
serde::Serialize,
clap::ValueEnum,
)] )]
pub(crate) enum AnimationFormat { pub(crate) enum AnimationFormat {
#[serde(rename = "apng")] #[serde(rename = "apng")]
@ -12,24 +22,16 @@ pub(crate) enum AnimationFormat {
Webp, Webp,
} }
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Deserialize)]
pub(crate) struct AnimationInput {
pub(crate) format: AnimationFormat,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Deserialize)] #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Deserialize)]
pub(crate) struct AnimationOutput { pub(crate) struct AnimationOutput {
pub(crate) format: AnimationFormat, pub(crate) format: AnimationFormat,
pub(crate) needs_transcode: bool, pub(crate) needs_transcode: bool,
} }
impl AnimationInput { impl AnimationFormat {
pub(crate) const fn build_output( pub(crate) const fn build_output(self, prescribed: Option<AnimationFormat>) -> AnimationOutput {
&self,
prescribed: Option<AnimationFormat>,
) -> AnimationOutput {
if let Some(prescribed) = prescribed { if let Some(prescribed) = prescribed {
let needs_transcode = !self.format.const_eq(prescribed); let needs_transcode = !self.const_eq(prescribed);
return AnimationOutput { return AnimationOutput {
format: prescribed, format: prescribed,
@ -38,13 +40,11 @@ impl AnimationInput {
} }
AnimationOutput { AnimationOutput {
format: self.format, format: self,
needs_transcode: false, needs_transcode: false,
} }
} }
}
impl AnimationFormat {
const fn const_eq(self, rhs: Self) -> bool { const fn const_eq(self, rhs: Self) -> bool {
match (self, rhs) { match (self, rhs) {
(Self::Apng, Self::Apng) (Self::Apng, Self::Apng)

View File

@ -18,7 +18,19 @@ pub(crate) enum OutputVideoFormat {
}, },
} }
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Deserialize)] #[derive(
Clone,
Copy,
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Hash,
serde::Deserialize,
serde::Serialize,
clap::ValueEnum,
)]
pub(crate) enum VideoCodec { pub(crate) enum VideoCodec {
#[serde(rename = "av1")] #[serde(rename = "av1")]
Av1, Av1,
@ -33,7 +45,17 @@ pub(crate) enum VideoCodec {
} }
#[derive( #[derive(
Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Deserialize, serde::Serialize, Clone,
Copy,
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Hash,
serde::Deserialize,
serde::Serialize,
clap::ValueEnum,
)] )]
pub(crate) enum AudioCodec { pub(crate) enum AudioCodec {
#[serde(rename = "aac")] #[serde(rename = "aac")]
@ -120,40 +142,92 @@ impl VideoFormat {
pub(crate) const fn build_output( pub(crate) const fn build_output(
self, self,
prescribed: Option<OutputVideoFormat>, video_codec: VideoCodec,
audio_codec: Option<AudioCodec>,
allow_audio: bool, allow_audio: bool,
) -> OutputVideoFormat { ) -> OutputVideoFormat {
match (prescribed, self) { match (video_codec, self) {
( (VideoCodec::Vp8, Self::Webm { alpha }) => OutputVideoFormat::Webm {
Some(OutputVideoFormat::Webm { video_codec: WebmCodec::Alpha(AlphaCodec {
video_codec: WebmCodec::Alpha(AlphaCodec { codec, .. }), alpha,
audio_codec, codec: WebmAlphaCodec::Vp8,
}), }),
Self::Webm { alpha }, audio_codec: if allow_audio {
) => OutputVideoFormat::Webm { match audio_codec {
video_codec: WebmCodec::Alpha(AlphaCodec { alpha, codec }), Some(AudioCodec::Vorbis) => Some(WebmAudioCodec::Vorbis),
audio_codec, _ => Some(WebmAudioCodec::Opus),
}, }
(Some(prescribed), _) => prescribed, } else {
(None, format) => match format { None
VideoFormat::Mp4 => OutputVideoFormat::Mp4 {
video_codec: Mp4Codec::H264,
audio_codec: if allow_audio {
Some(Mp4AudioCodec::Aac)
} else {
None
},
}, },
VideoFormat::Webm { alpha } => OutputVideoFormat::Webm { },
video_codec: WebmCodec::Alpha(AlphaCodec { (VideoCodec::Vp8, _) => OutputVideoFormat::Webm {
alpha, video_codec: WebmCodec::Alpha(AlphaCodec {
codec: WebmAlphaCodec::Vp9, alpha: false,
}), codec: WebmAlphaCodec::Vp8,
audio_codec: if allow_audio { }),
Some(WebmAudioCodec::Opus) audio_codec: if allow_audio {
} else { match audio_codec {
None Some(AudioCodec::Vorbis) => Some(WebmAudioCodec::Vorbis),
}, _ => Some(WebmAudioCodec::Opus),
}
} else {
None
},
},
(VideoCodec::Vp9, Self::Webm { alpha }) => OutputVideoFormat::Webm {
video_codec: WebmCodec::Alpha(AlphaCodec {
alpha,
codec: WebmAlphaCodec::Vp9,
}),
audio_codec: if allow_audio {
match audio_codec {
Some(AudioCodec::Vorbis) => Some(WebmAudioCodec::Vorbis),
_ => Some(WebmAudioCodec::Opus),
}
} else {
None
},
},
(VideoCodec::Vp9, _) => OutputVideoFormat::Webm {
video_codec: WebmCodec::Alpha(AlphaCodec {
alpha: false,
codec: WebmAlphaCodec::Vp9,
}),
audio_codec: if allow_audio {
match audio_codec {
Some(AudioCodec::Vorbis) => Some(WebmAudioCodec::Vorbis),
_ => Some(WebmAudioCodec::Opus),
}
} else {
None
},
},
(VideoCodec::Av1, _) => OutputVideoFormat::Webm {
video_codec: WebmCodec::Av1,
audio_codec: if allow_audio {
match audio_codec {
Some(AudioCodec::Vorbis) => Some(WebmAudioCodec::Vorbis),
_ => Some(WebmAudioCodec::Opus),
}
} else {
None
},
},
(VideoCodec::H264, _) => OutputVideoFormat::Mp4 {
video_codec: Mp4Codec::H264,
audio_codec: if allow_audio {
Some(Mp4AudioCodec::Aac)
} else {
None
},
},
(VideoCodec::H265, _) => OutputVideoFormat::Mp4 {
video_codec: Mp4Codec::H265,
audio_codec: if allow_audio {
Some(Mp4AudioCodec::Aac)
} else {
None
}, },
}, },
} }
@ -161,7 +235,7 @@ impl VideoFormat {
} }
impl OutputVideoFormat { impl OutputVideoFormat {
pub(super) const fn from_parts( pub(crate) const fn from_parts(
video_codec: VideoCodec, video_codec: VideoCodec,
audio_codec: Option<AudioCodec>, audio_codec: Option<AudioCodec>,
allow_audio: bool, allow_audio: bool,
@ -240,6 +314,13 @@ impl OutputVideoFormat {
} }
} }
pub(crate) const fn magick_format(self) -> &'static str {
match self {
Self::Mp4 { .. } => "MP4",
Self::Webm { .. } => "WEBM",
}
}
pub(crate) const fn ffmpeg_format(self) -> &'static str { pub(crate) const fn ffmpeg_format(self) -> &'static str {
match self { match self {
Self::Mp4 { .. } => "mp4", Self::Mp4 { .. } => "mp4",

View File

@ -2,10 +2,9 @@ use crate::{
bytes_stream::BytesStream, bytes_stream::BytesStream,
either::Either, either::Either,
error::{Error, UploadError}, error::{Error, UploadError},
formats::{InternalFormat, PrescribedFormats}, formats::{InternalFormat, Validations},
repo::{Alias, AliasRepo, AlreadyExists, DeleteToken, FullRepo, HashRepo}, repo::{Alias, AliasRepo, AlreadyExists, DeleteToken, FullRepo, HashRepo},
store::Store, store::Store,
CONFIG,
}; };
use actix_web::web::Bytes; use actix_web::web::Bytes;
use futures_util::{Stream, StreamExt}; use futures_util::{Stream, StreamExt};
@ -47,6 +46,7 @@ pub(crate) async fn ingest<R, S>(
store: &S, store: &S,
stream: impl Stream<Item = Result<Bytes, Error>> + Unpin + 'static, stream: impl Stream<Item = Result<Bytes, Error>> + Unpin + 'static,
declared_alias: Option<Alias>, declared_alias: Option<Alias>,
media: &crate::config::Media,
) -> Result<Session<R, S>, Error> ) -> Result<Session<R, S>, Error>
where where
R: FullRepo + 'static, R: FullRepo + 'static,
@ -57,18 +57,16 @@ where
let bytes = aggregate(stream).await?; let bytes = aggregate(stream).await?;
// TODO: load from config // TODO: load from config
let prescribed = PrescribedFormats { let prescribed = Validations {
image: None, image: &media.image,
animation: None, animation: &media.animation,
video: None, video: &media.video,
allow_audio: true,
}; };
tracing::trace!("Validating bytes"); tracing::trace!("Validating bytes");
let (input_type, validated_reader) = let (input_type, validated_reader) = crate::validate::validate_bytes(bytes, prescribed).await?;
crate::validate::validate_bytes(bytes, &prescribed).await?;
let processed_reader = if let Some(operations) = CONFIG.media.preprocess_steps() { let processed_reader = if let Some(operations) = media.preprocess_steps() {
if let Some(format) = input_type.processable_format() { if let Some(format) = input_type.processable_format() {
let (_, magick_args) = let (_, magick_args) =
crate::processor::build_chain(operations, format.file_extension())?; crate::processor::build_chain(operations, format.file_extension())?;

View File

@ -169,8 +169,10 @@ impl<R: FullRepo, S: Store + 'static> FormData for Upload<R, S> {
let stream = stream.map_err(Error::from); let stream = stream.map_err(Error::from);
Box::pin( Box::pin(
async move { ingest::ingest(&**repo, &**store, stream, None).await } async move {
.instrument(span), ingest::ingest(&**repo, &**store, stream, None, &CONFIG.media).await
}
.instrument(span),
) )
})), })),
) )
@ -221,6 +223,7 @@ impl<R: FullRepo, S: Store + 'static> FormData for Import<R, S> {
&**store, &**store,
stream, stream,
Some(Alias::from_existing(&filename)), Some(Alias::from_existing(&filename)),
&CONFIG.media,
) )
.await .await
} }
@ -475,7 +478,7 @@ async fn do_download_inline<R: FullRepo + 'static, S: Store + 'static>(
repo: web::Data<R>, repo: web::Data<R>,
store: web::Data<S>, store: web::Data<S>,
) -> Result<HttpResponse, Error> { ) -> Result<HttpResponse, Error> {
let mut session = ingest::ingest(&repo, &store, stream, None).await?; let mut session = ingest::ingest(&repo, &store, stream, None, &CONFIG.media).await?;
let alias = session.alias().expect("alias should exist").to_owned(); let alias = session.alias().expect("alias should exist").to_owned();
let delete_token = session.delete_token().await?; let delete_token = session.delete_token().await?;

View File

@ -6,6 +6,7 @@ use crate::{
repo::{Alias, DeleteToken, FullRepo, UploadId, UploadResult}, repo::{Alias, DeleteToken, FullRepo, UploadId, UploadResult},
serde_str::Serde, serde_str::Serde,
store::{Identifier, Store}, store::{Identifier, Store},
CONFIG,
}; };
use futures_util::TryStreamExt; use futures_util::TryStreamExt;
use std::path::PathBuf; use std::path::PathBuf;
@ -33,6 +34,7 @@ where
identifier, identifier,
Serde::into_inner(upload_id), Serde::into_inner(upload_id),
declared_alias.map(Serde::into_inner), declared_alias.map(Serde::into_inner),
&CONFIG.media,
) )
.await? .await?
} }
@ -69,6 +71,7 @@ async fn process_ingest<R, S>(
unprocessed_identifier: Vec<u8>, unprocessed_identifier: Vec<u8>,
upload_id: UploadId, upload_id: UploadId,
declared_alias: Option<Alias>, declared_alias: Option<Alias>,
media: &crate::config::Media,
) -> Result<(), Error> ) -> Result<(), Error>
where where
R: FullRepo + 'static, R: FullRepo + 'static,
@ -82,7 +85,7 @@ where
.await? .await?
.map_err(Error::from); .map_err(Error::from);
let session = crate::ingest::ingest(repo, store, stream, declared_alias).await?; let session = crate::ingest::ingest(repo, store, stream, declared_alias, media).await?;
let token = session.delete_token().await?; let token = session.delete_token().await?;

View File

@ -3,66 +3,235 @@ mod ffmpeg;
mod magick; mod magick;
use crate::{ use crate::{
discover::Discovery,
either::Either, either::Either,
error::Error, error::Error,
formats::{AnimationOutput, ImageOutput, InputFile, InternalFormat, PrescribedFormats}, formats::{
AnimationFormat, AnimationOutput, ImageInput, ImageOutput, InputFile, InternalFormat,
OutputVideoFormat, Validations, VideoFormat,
},
}; };
use actix_web::web::Bytes; use actix_web::web::Bytes;
use tokio::io::AsyncRead; use tokio::io::AsyncRead;
#[derive(Debug, thiserror::Error)]
pub(crate) enum ValidationError {
#[error("Too wide")]
Width,
#[error("Too tall")]
Height,
#[error("Too many pixels")]
Area,
#[error("Too many frames")]
Frames,
#[error("Filesize too large")]
Filesize,
#[error("Video is disabled")]
VideoDisabled,
}
const MEGABYTES: usize = 1024 * 1024;
#[tracing::instrument(skip_all)] #[tracing::instrument(skip_all)]
pub(crate) async fn validate_bytes( pub(crate) async fn validate_bytes(
bytes: Bytes, bytes: Bytes,
prescribed: &PrescribedFormats, validations: Validations<'_>,
) -> Result<(InternalFormat, impl AsyncRead + Unpin), Error> { ) -> Result<(InternalFormat, impl AsyncRead + Unpin), Error> {
let discovery = crate::discover::discover_bytes(bytes.clone()).await?; let Discovery {
input,
width,
height,
frames,
} = crate::discover::discover_bytes(bytes.clone()).await?;
match &discovery.input { match &input {
InputFile::Image(input) => { InputFile::Image(input) => {
let ImageOutput { let (format, read) =
format, process_image(bytes, *input, width, height, validations.image).await?;
needs_transcode,
} = input.build_output(prescribed.image);
let read = if needs_transcode { Ok((format, Either::left(read)))
Either::left(Either::left(magick::convert_image(
input.format,
format,
bytes,
)?))
} else {
Either::left(Either::right(exiftool::clear_metadata_bytes_read(bytes)?))
};
Ok((InternalFormat::Image(format), read))
} }
InputFile::Animation(input) => { InputFile::Animation(input) => {
let (format, read) = process_animation(
bytes,
*input,
width,
height,
frames.unwrap_or(1),
&validations,
)
.await?;
Ok((format, Either::right(Either::left(read))))
}
InputFile::Video(input) => {
let (format, read) = process_video(
bytes,
*input,
width,
height,
frames.unwrap_or(1),
validations.video,
)
.await?;
Ok((format, Either::right(Either::right(read))))
}
}
}
#[tracing::instrument(skip(bytes, validations))]
async fn process_image(
bytes: Bytes,
input: ImageInput,
width: u16,
height: u16,
validations: &crate::config::Image,
) -> Result<(InternalFormat, impl AsyncRead + Unpin), Error> {
if width > validations.max_width {
return Err(ValidationError::Width.into());
}
if height > validations.max_height {
return Err(ValidationError::Height.into());
}
if u32::from(width) * u32::from(height) > validations.max_area {
return Err(ValidationError::Area.into());
}
if bytes.len() > validations.max_file_size * MEGABYTES {
return Err(ValidationError::Filesize.into());
}
let ImageOutput {
format,
needs_transcode,
} = input.build_output(validations.format);
let read = if needs_transcode {
Either::left(magick::convert_image(input.format, format, bytes)?)
} else {
Either::right(exiftool::clear_metadata_bytes_read(bytes)?)
};
Ok((InternalFormat::Image(format), read))
}
fn validate_animation(
size: usize,
width: u16,
height: u16,
frames: u32,
validations: &crate::config::Animation,
) -> Result<(), ValidationError> {
if width > validations.max_width {
return Err(ValidationError::Width);
}
if height > validations.max_height {
return Err(ValidationError::Height);
}
if u32::from(width) * u32::from(height) > validations.max_area {
return Err(ValidationError::Area);
}
if frames > validations.max_frame_count {
return Err(ValidationError::Frames);
}
if size > validations.max_file_size * MEGABYTES {
return Err(ValidationError::Filesize);
}
Ok(())
}
#[tracing::instrument(skip(bytes, validations))]
async fn process_animation(
bytes: Bytes,
input: AnimationFormat,
width: u16,
height: u16,
frames: u32,
validations: &Validations<'_>,
) -> Result<(InternalFormat, impl AsyncRead + Unpin), Error> {
match validate_animation(bytes.len(), width, height, frames, validations.animation) {
Ok(()) => {
let AnimationOutput { let AnimationOutput {
format, format,
needs_transcode, needs_transcode,
} = input.build_output(prescribed.animation); } = input.build_output(validations.animation.format);
let read = if needs_transcode { let read = if needs_transcode {
Either::right(Either::left(magick::convert_animation( Either::left(magick::convert_animation(input, format, bytes)?)
input.format,
format,
bytes,
)?))
} else { } else {
Either::right(Either::right(Either::left( Either::right(Either::left(exiftool::clear_metadata_bytes_read(bytes)?))
exiftool::clear_metadata_bytes_read(bytes)?,
)))
}; };
Ok((InternalFormat::Animation(format), read)) Ok((InternalFormat::Animation(format), read))
} }
InputFile::Video(input) => { Err(_) if validate_video(bytes.len(), width, height, frames, validations.video).is_ok() => {
let output = input.build_output(prescribed.video, prescribed.allow_audio); let output = OutputVideoFormat::from_parts(
let read = Either::right(Either::right(Either::right( validations.video.video_codec,
ffmpeg::transcode_bytes(*input, output, bytes).await?, validations.video.audio_codec,
))); validations.video.allow_audio,
);
let read = Either::right(Either::right(magick::convert_video(input, output, bytes)?));
Ok((InternalFormat::Video(output.internal_format()), read)) Ok((InternalFormat::Video(output.internal_format()), read))
} }
Err(e) => Err(e.into()),
} }
} }
fn validate_video(
size: usize,
width: u16,
height: u16,
frames: u32,
validations: &crate::config::Video,
) -> Result<(), ValidationError> {
if !validations.enable {
return Err(ValidationError::VideoDisabled);
}
if width > validations.max_width {
return Err(ValidationError::Width);
}
if height > validations.max_height {
return Err(ValidationError::Height);
}
if u32::from(width) * u32::from(height) > validations.max_area {
return Err(ValidationError::Area);
}
if frames > validations.max_frame_count {
return Err(ValidationError::Frames);
}
if size > validations.max_file_size * MEGABYTES {
return Err(ValidationError::Filesize);
}
Ok(())
}
#[tracing::instrument(skip(bytes, validations))]
async fn process_video(
bytes: Bytes,
input: VideoFormat,
width: u16,
height: u16,
frames: u32,
validations: &crate::config::Video,
) -> Result<(InternalFormat, impl AsyncRead + Unpin), Error> {
validate_video(bytes.len(), width, height, frames, validations)?;
let output = input.build_output(
validations.video_codec,
validations.audio_codec,
validations.allow_audio,
);
let read = ffmpeg::transcode_bytes(input, output, bytes).await?;
Ok((InternalFormat::Video(output.internal_format()), read))
}

View File

@ -2,7 +2,7 @@ use actix_web::web::Bytes;
use tokio::io::AsyncRead; use tokio::io::AsyncRead;
use crate::{ use crate::{
formats::{AnimationFormat, ImageFormat}, formats::{AnimationFormat, ImageFormat, OutputVideoFormat},
magick::MagickError, magick::MagickError,
process::Process, process::Process,
}; };
@ -37,3 +37,17 @@ pub(super) fn convert_animation(
Ok(process.bytes_read(bytes)) Ok(process.bytes_read(bytes))
} }
pub(super) fn convert_video(
input: AnimationFormat,
output: OutputVideoFormat,
bytes: Bytes,
) -> Result<impl AsyncRead + Unpin, MagickError> {
let input_arg = format!("{}:-", input.magick_format());
let output_arg = format!("{}:-", output.magick_format());
let process = Process::run("magick", &["-strip", &input_arg, "-coalesce", &output_arg])
.map_err(MagickError::Process)?;
Ok(process.bytes_read(bytes))
}