LemmyNet/pict-rs
2
0
Fork 0
mirror of https://git.asonix.dog/asonix/pict-rs synced 2024-12-22 19:31:35 +00:00
Code Releases Activity

Attempt to clean stray magick files

Browse source 
This change sets a unique temp directory for each invocation of imagemagick and spawns a task to remove that directory after the command terminates
This commit is contained in:
asonix 2023-11-09 18:20:59 -06:00
parent ee5bfd6557
commit f61dac8187
13 changed files with 299 additions and 144 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/discover/exiftool.rs
View file

@ -42,7 +42,7 @@ pub(super) async fn check_reorient(
#[tracing::instrument(level = "trace", skip(input))] #[tracing::instrument(level = "trace", skip(input))]
async fn needs_reorienting(input: Bytes, timeout: u64) -> Result<bool, ExifError> { async fn needs_reorienting(input: Bytes, timeout: u64) -> Result<bool, ExifError> {
let process = Process::run("exiftool", &["-n", "-Orientation", "-"], timeout)?; let process = Process::run("exiftool", &["-n", "-Orientation", "-"], &[], timeout)?;
let mut reader = process.bytes_read(input); let mut reader = process.bytes_read(input);
let mut buf = String::new(); let mut buf = String::new();

28
src/discover/ffmpeg.rs
View file

@ -201,7 +201,6 @@ where
Fut: std::future::Future<Output = Result<crate::file::File, FfMpegError>>, Fut: std::future::Future<Output = Result<crate::file::File, FfMpegError>>,
{ {
let input_file = tmp_dir.tmp_file(None); let input_file = tmp_dir.tmp_file(None);
let input_file_str = input_file.to_str().ok_or(FfMpegError::Path)?;
crate::store::file_store::safe_create_parent(&input_file) crate::store::file_store::safe_create_parent(&input_file)
.await .await
.map_err(FfMpegError::CreateDir)?; .map_err(FfMpegError::CreateDir)?;
@ -215,17 +214,18 @@ where
let process = Process::run( let process = Process::run(
"ffprobe", "ffprobe",
&[ &[
"-v", "-v".as_ref(),
"quiet", "quiet".as_ref(),
"-count_frames", "-count_frames".as_ref(),
"-show_entries", "-show_entries".as_ref(),
"stream=width,height,nb_read_frames,codec_name,pix_fmt:format=format_name", "stream=width,height,nb_read_frames,codec_name,pix_fmt:format=format_name".as_ref(),
"-of", "-of".as_ref(),
"default=noprint_wrappers=1:nokey=1", "default=noprint_wrappers=1:nokey=1".as_ref(),
"-print_format", "-print_format".as_ref(),
"json", "json".as_ref(),
input_file_str, input_file.as_os_str(),
], ],
&[],
timeout, timeout,
)?; )?;
@ -235,9 +235,8 @@ where
.read_to_end(&mut output) .read_to_end(&mut output)
.await .await
.map_err(FfMpegError::Read)?; .map_err(FfMpegError::Read)?;
tokio::fs::remove_file(input_file_str)
.await drop(input_file);
.map_err(FfMpegError::RemoveFile)?;
let output: FfMpegDiscovery = serde_json::from_slice(&output).map_err(FfMpegError::Json)?; let output: FfMpegDiscovery = serde_json::from_slice(&output).map_err(FfMpegError::Json)?;
@ -273,6 +272,7 @@ async fn alpha_pixel_formats(timeout: u64) -> Result<HashSet<String>, FfMpegErro
"-print_format", "-print_format",
"json", "json",
], ],
&[],
timeout, timeout,
)?; )?;

45
src/discover/magick.rs
View file

@ -7,7 +7,7 @@ use tokio::io::AsyncReadExt;
use crate::{ use crate::{
discover::DiscoverError, discover::DiscoverError,
formats::{AnimationFormat, ImageFormat, ImageInput, InputFile}, formats::{AnimationFormat, ImageFormat, ImageInput, InputFile},
magick::MagickError, magick::{MagickError, MAGICK_TEMPORARY_PATH},
process::Process, process::Process,
tmp_file::TmpDir, tmp_file::TmpDir,
}; };
@ -99,8 +99,12 @@ where
F: FnOnce(crate::file::File) -> Fut, F: FnOnce(crate::file::File) -> Fut,
Fut: std::future::Future<Output = Result<crate::file::File, MagickError>>, Fut: std::future::Future<Output = Result<crate::file::File, MagickError>>,
{ {
let temporary_path = tmp_dir
.tmp_folder()
.await
.map_err(MagickError::CreateTemporaryDirectory)?;
let input_file = tmp_dir.tmp_file(None); let input_file = tmp_dir.tmp_file(None);
let input_file_str = input_file.to_str().ok_or(MagickError::Path)?;
crate::store::file_store::safe_create_parent(&input_file) crate::store::file_store::safe_create_parent(&input_file)
.await .await
.map_err(MagickError::CreateDir)?; .map_err(MagickError::CreateDir)?;
@ -111,9 +115,17 @@ where
let tmp_one = (f)(tmp_one).await?; let tmp_one = (f)(tmp_one).await?;
tmp_one.close().await.map_err(MagickError::CloseFile)?; tmp_one.close().await.map_err(MagickError::CloseFile)?;
let envs = [(MAGICK_TEMPORARY_PATH, temporary_path.as_os_str())];
let process = Process::run( let process = Process::run(
"magick", "magick",
&["convert", "-ping", input_file_str, "INFO:"], &[
"convert".as_ref(),
"-ping".as_ref(),
input_file.as_os_str(),
"INFO:".as_ref(),
],
&envs,
timeout, timeout,
)?; )?;
@ -123,9 +135,9 @@ where
.read_to_string(&mut output) .read_to_string(&mut output)
.await .await
.map_err(MagickError::Read)?; .map_err(MagickError::Read)?;
tokio::fs::remove_file(input_file_str)
.await drop(input_file);
.map_err(MagickError::RemoveFile)?; drop(temporary_path);
if output.is_empty() { if output.is_empty() {
return Err(MagickError::Empty); return Err(MagickError::Empty);
@ -154,8 +166,12 @@ where
F: FnOnce(crate::file::File) -> Fut, F: FnOnce(crate::file::File) -> Fut,
Fut: std::future::Future<Output = Result<crate::file::File, MagickError>>, Fut: std::future::Future<Output = Result<crate::file::File, MagickError>>,
{ {
let temporary_path = tmp_dir
.tmp_folder()
.await
.map_err(MagickError::CreateTemporaryDirectory)?;
let input_file = tmp_dir.tmp_file(None); let input_file = tmp_dir.tmp_file(None);
let input_file_str = input_file.to_str().ok_or(MagickError::Path)?;
crate::store::file_store::safe_create_parent(&input_file) crate::store::file_store::safe_create_parent(&input_file)
.await .await
.map_err(MagickError::CreateDir)?; .map_err(MagickError::CreateDir)?;
@ -166,9 +182,17 @@ where
let tmp_one = (f)(tmp_one).await?; let tmp_one = (f)(tmp_one).await?;
tmp_one.close().await.map_err(MagickError::CloseFile)?; tmp_one.close().await.map_err(MagickError::CloseFile)?;
let envs = [(MAGICK_TEMPORARY_PATH, temporary_path.as_os_str())];
let process = Process::run( let process = Process::run(
"magick", "magick",
&["convert", "-ping", input_file_str, "JSON:"], &[
"convert".as_ref(),
"-ping".as_ref(),
input_file.as_os_str(),
"JSON:".as_ref(),
],
&envs,
timeout, timeout,
)?; )?;
@ -178,9 +202,8 @@ where
.read_to_end(&mut output) .read_to_end(&mut output)
.await .await
.map_err(MagickError::Read)?; .map_err(MagickError::Read)?;
tokio::fs::remove_file(input_file_str)
.await drop(input_file);
.map_err(MagickError::RemoveFile)?;
if output.is_empty() { if output.is_empty() {
return Err(MagickError::Empty); return Err(MagickError::Empty);

4
src/exiftool.rs
View file

@ -45,7 +45,7 @@ impl ExifError {
#[tracing::instrument(level = "trace", skip(input))] #[tracing::instrument(level = "trace", skip(input))]
pub(crate) async fn needs_reorienting(timeout: u64, input: Bytes) -> Result<bool, ExifError> { pub(crate) async fn needs_reorienting(timeout: u64, input: Bytes) -> Result<bool, ExifError> {
let process = Process::run("exiftool", &["-n", "-Orientation", "-"], timeout)?; let process = Process::run("exiftool", &["-n", "-Orientation", "-"], &[], timeout)?;
let mut reader = process.bytes_read(input); let mut reader = process.bytes_read(input);
let mut buf = String::new(); let mut buf = String::new();
@ -62,7 +62,7 @@ pub(crate) fn clear_metadata_bytes_read(
timeout: u64, timeout: u64,
input: Bytes, input: Bytes,
) -> Result<impl AsyncRead + Unpin, ExifError> { ) -> Result<impl AsyncRead + Unpin, ExifError> {
let process = Process::run("exiftool", &["-all=", "-", "-out", "-"], timeout)?; let process = Process::run("exiftool", &["-all=", "-", "-out", "-"], &[], timeout)?;
Ok(process.bytes_read(input)) Ok(process.bytes_read(input))
} }

10
src/ffmpeg.rs
View file

@ -29,17 +29,11 @@ pub(crate) enum FfMpegError {
#[error("Error closing file")] #[error("Error closing file")]
CloseFile(#[source] std::io::Error), CloseFile(#[source] std::io::Error),
#[error("Error removing file")]
RemoveFile(#[source] std::io::Error),
#[error("Error in store")] #[error("Error in store")]
Store(#[source] StoreError), Store(#[source] StoreError),
#[error("Invalid media file provided")] #[error("Invalid media file provided")]
CommandFailed(ProcessError), CommandFailed(ProcessError),
#[error("Invalid file path")]
Path,
} }
impl From<ProcessError> for FfMpegError { impl From<ProcessError> for FfMpegError {
@ -64,9 +58,7 @@ impl FfMpegError {
| Self::ReadFile(_) | Self::ReadFile(_)
| Self::OpenFile(_) | Self::OpenFile(_)
| Self::CreateFile(_) | Self::CreateFile(_)
| Self::CloseFile(_) | Self::CloseFile(_) => ErrorCode::COMMAND_ERROR,
| Self::RemoveFile(_)
| Self::Path => ErrorCode::COMMAND_ERROR,
} }
} }

33
src/generate/ffmpeg.rs
View file

@ -55,13 +55,11 @@ pub(super) async fn thumbnail<S: Store>(
timeout: u64, timeout: u64,
) -> Result<BoxRead<'static>, FfMpegError> { ) -> Result<BoxRead<'static>, FfMpegError> {
let input_file = tmp_dir.tmp_file(Some(input_format.file_extension())); let input_file = tmp_dir.tmp_file(Some(input_format.file_extension()));
let input_file_str = input_file.to_str().ok_or(FfMpegError::Path)?;
crate::store::file_store::safe_create_parent(&input_file) crate::store::file_store::safe_create_parent(&input_file)
.await .await
.map_err(FfMpegError::CreateDir)?; .map_err(FfMpegError::CreateDir)?;
let output_file = tmp_dir.tmp_file(Some(format.to_file_extension())); let output_file = tmp_dir.tmp_file(Some(format.to_file_extension()));
let output_file_str = output_file.to_str().ok_or(FfMpegError::Path)?;
crate::store::file_store::safe_create_parent(&output_file) crate::store::file_store::safe_create_parent(&output_file)
.await .await
.map_err(FfMpegError::CreateDir)?; .map_err(FfMpegError::CreateDir)?;
@ -82,26 +80,25 @@ pub(super) async fn thumbnail<S: Store>(
let process = Process::run( let process = Process::run(
"ffmpeg", "ffmpeg",
&[ &[
"-hide_banner", "-hide_banner".as_ref(),
"-v", "-v".as_ref(),
"warning", "warning".as_ref(),
"-i", "-i".as_ref(),
input_file_str, input_file.as_os_str(),
"-frames:v", "-frames:v".as_ref(),
"1", "1".as_ref(),
"-codec", "-codec".as_ref(),
format.as_ffmpeg_codec(), format.as_ffmpeg_codec().as_ref(),
"-f", "-f".as_ref(),
format.as_ffmpeg_format(), format.as_ffmpeg_format().as_ref(),
output_file_str, output_file.as_os_str(),
], ],
&[],
timeout, timeout,
)?; )?;
process.wait().await?; process.wait().await?;
tokio::fs::remove_file(input_file) drop(input_file);
.await
.map_err(FfMpegError::RemoveFile)?;
let tmp_two = crate::file::File::open(&output_file) let tmp_two = crate::file::File::open(&output_file)
.await .await
@ -111,7 +108,7 @@ pub(super) async fn thumbnail<S: Store>(
.await .await
.map_err(FfMpegError::ReadFile)?; .map_err(FfMpegError::ReadFile)?;
let reader = tokio_util::io::StreamReader::new(stream); let reader = tokio_util::io::StreamReader::new(stream);
let clean_reader = crate::tmp_file::cleanup_tmpfile(reader, output_file); let clean_reader = output_file.reader(reader);
Ok(Box::pin(clean_reader)) Ok(Box::pin(clean_reader))
} }

37
src/generate/magick.rs
View file

@ -1,7 +1,11 @@
use std::sync::Arc; use std::{ffi::OsStr, sync::Arc};
use crate::{ use crate::{
formats::ProcessableFormat, magick::MagickError, process::Process, read::BoxRead, store::Store, formats::ProcessableFormat,
magick::{MagickError, MAGICK_TEMPORARY_PATH},
process::Process,
read::BoxRead,
store::Store,
tmp_file::TmpDir, tmp_file::TmpDir,
}; };
@ -17,8 +21,12 @@ where
F: FnOnce(crate::file::File) -> Fut, F: FnOnce(crate::file::File) -> Fut,
Fut: std::future::Future<Output = Result<crate::file::File, MagickError>>, Fut: std::future::Future<Output = Result<crate::file::File, MagickError>>,
{ {
let temporary_path = tmp_dir
.tmp_folder()
.await
.map_err(MagickError::CreateTemporaryDirectory)?;
let input_file = tmp_dir.tmp_file(None); let input_file = tmp_dir.tmp_file(None);
let input_file_str = input_file.to_str().ok_or(MagickError::Path)?;
crate::store::file_store::safe_create_parent(&input_file) crate::store::file_store::safe_create_parent(&input_file)
.await .await
.map_err(MagickError::CreateDir)?; .map_err(MagickError::CreateDir)?;
@ -29,26 +37,33 @@ where
let tmp_one = (write_file)(tmp_one).await?; let tmp_one = (write_file)(tmp_one).await?;
tmp_one.close().await.map_err(MagickError::CloseFile)?; tmp_one.close().await.map_err(MagickError::CloseFile)?;
let input_arg = format!("{}:{input_file_str}[0]", input_format.magick_format()); let input_arg = [
input_format.magick_format().as_ref(),
input_file.as_os_str(),
]
.join(":".as_ref());
let output_arg = format!("{}:-", format.magick_format()); let output_arg = format!("{}:-", format.magick_format());
let quality = quality.map(|q| q.to_string()); let quality = quality.map(|q| q.to_string());
let len = 3 + if format.coalesce() { 1 } else { 0 } + if quality.is_some() { 1 } else { 0 }; let len = 3 + if format.coalesce() { 1 } else { 0 } + if quality.is_some() { 1 } else { 0 };
let mut args: Vec<&str> = Vec::with_capacity(len); let mut args: Vec<&OsStr> = Vec::with_capacity(len);
args.push("convert"); args.push("convert".as_ref());
args.push(&input_arg); args.push(&input_arg);
if format.coalesce() { if format.coalesce() {
args.push("-coalesce"); args.push("-coalesce".as_ref());
} }
if let Some(quality) = &quality { if let Some(quality) = &quality {
args.extend(["-quality", quality]); args.extend(["-quality".as_ref(), quality.as_ref()] as [&OsStr; 2]);
} }
args.push(&output_arg); args.push(output_arg.as_ref());
let reader = Process::run("magick", &args, timeout)?.read(); let envs = [(MAGICK_TEMPORARY_PATH, temporary_path.as_os_str())];
let clean_reader = crate::tmp_file::cleanup_tmpfile(reader, input_file); let reader = Process::run("magick", &args, &envs, timeout)?.read();
let clean_reader = input_file.reader(reader);
let clean_reader = temporary_path.reader(clean_reader);
Ok(Box::pin(clean_reader)) Ok(Box::pin(clean_reader))
} }

49
src/magick.rs
View file

@ -1,4 +1,4 @@
use std::sync::Arc; use std::{ffi::OsStr, sync::Arc};
use crate::{ use crate::{
error_code::ErrorCode, error_code::ErrorCode,
@ -11,6 +11,8 @@ use crate::{
use tokio::io::AsyncRead; use tokio::io::AsyncRead;
pub(crate) const MAGICK_TEMPORARY_PATH: &str = "MAGICK_TEMPORARY_PATH";
#[derive(Debug, thiserror::Error)] #[derive(Debug, thiserror::Error)]
pub(crate) enum MagickError { pub(crate) enum MagickError {
#[error("Error in imagemagick process")] #[error("Error in imagemagick process")]
@ -34,12 +36,12 @@ pub(crate) enum MagickError {
#[error("Error creating directory")] #[error("Error creating directory")]
CreateDir(#[source] crate::store::file_store::FileError), CreateDir(#[source] crate::store::file_store::FileError),
#[error("Error creating temporary directory")]
CreateTemporaryDirectory(#[source] std::io::Error),
#[error("Error closing file")] #[error("Error closing file")]
CloseFile(#[source] std::io::Error), CloseFile(#[source] std::io::Error),
#[error("Error removing file")]
RemoveFile(#[source] std::io::Error),
#[error("Error in metadata discovery")] #[error("Error in metadata discovery")]
Discover(#[source] crate::discover::DiscoverError), Discover(#[source] crate::discover::DiscoverError),
@ -48,9 +50,6 @@ pub(crate) enum MagickError {
#[error("Command output is empty")] #[error("Command output is empty")]
Empty, Empty,
#[error("Invalid file path")]
Path,
} }
impl From<ProcessError> for MagickError { impl From<ProcessError> for MagickError {
@ -73,11 +72,10 @@ impl MagickError {
| Self::Write(_) | Self::Write(_)
| Self::CreateFile(_) | Self::CreateFile(_)
| Self::CreateDir(_) | Self::CreateDir(_)
| Self::CreateTemporaryDirectory(_)
| Self::CloseFile(_) | Self::CloseFile(_)
| Self::RemoveFile(_)
| Self::Discover(_) | Self::Discover(_)
| Self::Empty | Self::Empty => ErrorCode::COMMAND_ERROR,
| Self::Path => ErrorCode::COMMAND_ERROR,
} }
} }
@ -103,8 +101,12 @@ where
F: FnOnce(crate::file::File) -> Fut, F: FnOnce(crate::file::File) -> Fut,
Fut: std::future::Future<Output = Result<crate::file::File, MagickError>>, Fut: std::future::Future<Output = Result<crate::file::File, MagickError>>,
{ {
let temporary_path = tmp_dir
.tmp_folder()
.await
.map_err(MagickError::CreateTemporaryDirectory)?;
let input_file = tmp_dir.tmp_file(None); let input_file = tmp_dir.tmp_file(None);
let input_file_str = input_file.to_str().ok_or(MagickError::Path)?;
crate::store::file_store::safe_create_parent(&input_file) crate::store::file_store::safe_create_parent(&input_file)
.await .await
.map_err(MagickError::CreateDir)?; .map_err(MagickError::CreateDir)?;
@ -115,7 +117,11 @@ where
let tmp_one = (write_file)(tmp_one).await?; let tmp_one = (write_file)(tmp_one).await?;
tmp_one.close().await.map_err(MagickError::CloseFile)?; tmp_one.close().await.map_err(MagickError::CloseFile)?;
let input_arg = format!("{}:{input_file_str}", input_format.magick_format()); let input_arg = [
input_format.magick_format().as_ref(),
input_file.as_os_str(),
]
.join(":".as_ref());
let output_arg = format!("{}:-", format.magick_format()); let output_arg = format!("{}:-", format.magick_format());
let quality = quality.map(|q| q.to_string()); let quality = quality.map(|q| q.to_string());
@ -124,21 +130,24 @@ where
+ if quality.is_some() { 1 } else { 0 } + if quality.is_some() { 1 } else { 0 }
+ process_args.len(); + process_args.len();
let mut args: Vec<&str> = Vec::with_capacity(len); let mut args: Vec<&OsStr> = Vec::with_capacity(len);
args.push("convert"); args.push("convert".as_ref());
args.push(&input_arg); args.push(&input_arg);
if input_format.coalesce() { if input_format.coalesce() {
args.push("-coalesce"); args.push("-coalesce".as_ref());
} }
args.extend(process_args.iter().map(|s| s.as_str())); args.extend(process_args.iter().map(|s| AsRef::<OsStr>::as_ref(s)));
if let Some(quality) = &quality { if let Some(quality) = &quality {
args.extend(["-quality", quality]); args.extend(["-quality".as_ref(), quality.as_ref()] as [&OsStr; 2]);
} }
args.push(&output_arg); args.push(output_arg.as_ref());
let reader = Process::run("magick", &args, timeout)?.read(); let envs = [(MAGICK_TEMPORARY_PATH, temporary_path.as_os_str())];
let clean_reader = crate::tmp_file::cleanup_tmpfile(reader, input_file); let reader = Process::run("magick", &args, &envs, timeout)?.read();
let clean_reader = input_file.reader(reader);
let clean_reader = temporary_path.reader(clean_reader);
Ok(Box::pin(clean_reader)) Ok(Box::pin(clean_reader))
} }

22
src/process.rs
View file

@ -1,6 +1,7 @@
use actix_web::web::Bytes; use actix_web::web::Bytes;
use flume::r#async::RecvFut; use flume::r#async::RecvFut;
use std::{ use std::{
ffi::OsStr,
future::Future, future::Future,
pin::Pin, pin::Pin,
process::{ExitStatus, Stdio}, process::{ExitStatus, Stdio},
@ -115,9 +116,24 @@ impl ProcessError {
} }
impl Process { impl Process {
pub(crate) fn run(command: &str, args: &[&str], timeout: u64) -> Result<Self, ProcessError> { pub(crate) fn run<T>(
let res = tracing::trace_span!(parent: None, "Create command", %command) command: &str,
.in_scope(|| Self::spawn(command, Command::new(command).args(args), timeout)); args: &[T],
envs: &[(&str, &OsStr)],
timeout: u64,
) -> Result<Self, ProcessError>
where
T: AsRef<OsStr>,
{
let res = tracing::trace_span!(parent: None, "Create command", %command).in_scope(|| {
Self::spawn(
command,
Command::new(command)
.args(args)
.envs(envs.into_iter().copied()),
timeout,
)
});
match res { match res {
Ok(this) => Ok(this), Ok(this) => Ok(this),

120
src/tmp_file.rs
View file

@ -1,4 +1,8 @@
use std::{path::PathBuf, sync::Arc}; use std::{
ops::Deref,
path::{Path, PathBuf},
sync::Arc,
};
use tokio::io::AsyncRead; use tokio::io::AsyncRead;
use uuid::Uuid; use uuid::Uuid;
@ -16,20 +20,33 @@ impl TmpDir {
Ok(Arc::new(TmpDir { path: Some(path) })) Ok(Arc::new(TmpDir { path: Some(path) }))
} }
pub(crate) fn tmp_file(&self, ext: Option<&str>) -> PathBuf { fn build_tmp_file(&self, ext: Option<&str>) -> Arc<Path> {
if let Some(ext) = ext { if let Some(ext) = ext {
self.path Arc::from(self.path.as_ref().expect("tmp path exists").join(format!(
.as_ref() "{}{}",
.expect("tmp path exists") Uuid::new_v4(),
.join(format!("{}{}", Uuid::new_v4(), ext)) ext
)))
} else { } else {
self.path Arc::from(
.as_ref() self.path
.expect("tmp path exists") .as_ref()
.join(Uuid::new_v4().to_string()) .expect("tmp path exists")
.join(Uuid::new_v4().to_string()),
)
} }
} }
pub(crate) fn tmp_file(&self, ext: Option<&str>) -> TmpFile {
TmpFile(self.build_tmp_file(ext))
}
pub(crate) async fn tmp_folder(&self) -> std::io::Result<TmpFolder> {
let path = self.build_tmp_file(None);
tokio::fs::create_dir(&path).await?;
Ok(TmpFolder(path))
}
pub(crate) async fn cleanup(self: Arc<Self>) -> std::io::Result<()> { pub(crate) async fn cleanup(self: Arc<Self>) -> std::io::Result<()> {
if let Some(path) = Arc::into_inner(self).and_then(|mut this| this.path.take()) { if let Some(path) = Arc::into_inner(self).and_then(|mut this| this.path.take()) {
tokio::fs::remove_dir_all(path).await?; tokio::fs::remove_dir_all(path).await?;
@ -47,7 +64,66 @@ impl Drop for TmpDir {
} }
} }
struct TmpFile(PathBuf); #[must_use]
pub(crate) struct TmpFolder(Arc<Path>);
impl TmpFolder {
pub(crate) fn reader<R: AsyncRead>(self, reader: R) -> TmpFolderCleanup<R> {
TmpFolderCleanup {
inner: reader,
folder: self,
}
}
}
impl AsRef<Path> for TmpFolder {
fn as_ref(&self) -> &Path {
&self.0
}
}
impl Deref for TmpFolder {
type Target = Path;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl Drop for TmpFolder {
fn drop(&mut self) {
crate::sync::spawn(
"remove-tmpfolder",
tokio::fs::remove_dir_all(self.0.clone()),
);
}
}
#[must_use]
pub(crate) struct TmpFile(Arc<Path>);
impl TmpFile {
pub(crate) fn reader<R: AsyncRead>(self, reader: R) -> TmpFileCleanup<R> {
TmpFileCleanup {
inner: reader,
file: self,
}
}
}
impl AsRef<Path> for TmpFile {
fn as_ref(&self) -> &Path {
&self.0
}
}
impl Deref for TmpFile {
type Target = Path;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl Drop for TmpFile { impl Drop for TmpFile {
fn drop(&mut self) { fn drop(&mut self) {
@ -64,10 +140,12 @@ pin_project_lite::pin_project! {
} }
} }
pub(crate) fn cleanup_tmpfile<R: AsyncRead>(reader: R, file: PathBuf) -> TmpFileCleanup<R> { pin_project_lite::pin_project! {
TmpFileCleanup { pub(crate) struct TmpFolderCleanup<R> {
inner: reader, #[pin]
file: TmpFile(file), inner: R,
folder: TmpFolder,
} }
} }
@ -82,3 +160,15 @@ impl<R: AsyncRead> AsyncRead for TmpFileCleanup<R> {
this.inner.poll_read(cx, buf) this.inner.poll_read(cx, buf)
} }
} }
impl<R: AsyncRead> AsyncRead for TmpFolderCleanup<R> {
fn poll_read(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
buf: &mut tokio::io::ReadBuf<'_>,
) -> std::task::Poll<std::io::Result<()>> {
let this = self.as_mut().project();
this.inner.poll_read(cx, buf)
}
}

2
src/validate/exiftool.rs
View file

@ -7,7 +7,7 @@ pub(crate) fn clear_metadata_bytes_read(
input: Bytes, input: Bytes,
timeout: u64, timeout: u64,
) -> Result<BoxRead<'static>, ExifError> { ) -> Result<BoxRead<'static>, ExifError> {
let process = Process::run("exiftool", &["-all=", "-", "-out", "-"], timeout)?; let process = Process::run("exiftool", &["-all=", "-", "-out", "-"], &[], timeout)?;
Ok(Box::pin(process.bytes_read(input))) Ok(Box::pin(process.bytes_read(input)))
} }

62
src/validate/ffmpeg.rs
View file

@ -1,3 +1,5 @@
use std::ffi::OsStr;
use actix_web::web::Bytes; use actix_web::web::Bytes;
use crate::{ use crate::{
@ -17,7 +19,6 @@ pub(super) async fn transcode_bytes(
bytes: Bytes, bytes: Bytes,
) -> Result<BoxRead<'static>, FfMpegError> { ) -> Result<BoxRead<'static>, FfMpegError> {
let input_file = tmp_dir.tmp_file(None); let input_file = tmp_dir.tmp_file(None);
let input_file_str = input_file.to_str().ok_or(FfMpegError::Path)?;
crate::store::file_store::safe_create_parent(&input_file) crate::store::file_store::safe_create_parent(&input_file)
.await .await
.map_err(FfMpegError::CreateDir)?; .map_err(FfMpegError::CreateDir)?;
@ -32,12 +33,11 @@ pub(super) async fn transcode_bytes(
tmp_one.close().await.map_err(FfMpegError::CloseFile)?; tmp_one.close().await.map_err(FfMpegError::CloseFile)?;
let output_file = tmp_dir.tmp_file(None); let output_file = tmp_dir.tmp_file(None);
let output_file_str = output_file.to_str().ok_or(FfMpegError::Path)?;
transcode_files( transcode_files(
input_file_str, input_file.as_os_str(),
input_format, input_format,
output_file_str, output_file.as_os_str(),
output_format, output_format,
crf, crf,
timeout, timeout,
@ -52,15 +52,15 @@ pub(super) async fn transcode_bytes(
.await .await
.map_err(FfMpegError::ReadFile)?; .map_err(FfMpegError::ReadFile)?;
let reader = tokio_util::io::StreamReader::new(stream); let reader = tokio_util::io::StreamReader::new(stream);
let clean_reader = crate::tmp_file::cleanup_tmpfile(reader, output_file); let clean_reader = output_file.reader(reader);
Ok(Box::pin(clean_reader)) Ok(Box::pin(clean_reader))
} }
async fn transcode_files( async fn transcode_files(
input_path: &str, input_path: &OsStr,
input_format: InputVideoFormat, input_format: InputVideoFormat,
output_path: &str, output_path: &OsStr,
output_format: OutputVideo, output_format: OutputVideo,
crf: u8, crf: u8,
timeout: u64, timeout: u64,
@ -74,47 +74,51 @@ async fn transcode_files(
} = output_format; } = output_format;
let mut args = vec![ let mut args = vec![
"-hide_banner", "-hide_banner".as_ref(),
"-v", "-v".as_ref(),
"warning", "warning".as_ref(),
"-f", "-f".as_ref(),
input_format.ffmpeg_format(), input_format.ffmpeg_format().as_ref(),
"-i", "-i".as_ref(),
input_path, input_path,
]; ];
if transcode_video { if transcode_video {
args.extend([ args.extend([
"-pix_fmt", "-pix_fmt".as_ref(),
output_format.pix_fmt(), output_format.pix_fmt().as_ref(),
"-vf", "-vf".as_ref(),
"scale=trunc(iw/2)*2:trunc(ih/2)*2", "scale=trunc(iw/2)*2:trunc(ih/2)*2".as_ref(),
"-c:v", "-c:v".as_ref(),
output_format.ffmpeg_video_codec(), output_format.ffmpeg_video_codec().as_ref(),
"-crf", "-crf".as_ref(),
&crf, &crf.as_ref(),
]); ] as [&OsStr; 8]);
if output_format.is_vp9() { if output_format.is_vp9() {
args.extend(["-b:v", "0"]); args.extend(["-b:v".as_ref(), "0".as_ref()] as [&OsStr; 2]);
} }
} else { } else {
args.extend(["-c:v", "copy"]); args.extend(["-c:v".as_ref(), "copy".as_ref()] as [&OsStr; 2]);
} }
if transcode_audio { if transcode_audio {
if let Some(audio_codec) = output_format.ffmpeg_audio_codec() { if let Some(audio_codec) = output_format.ffmpeg_audio_codec() {
args.extend(["-c:a", audio_codec]); args.extend(["-c:a".as_ref(), audio_codec.as_ref()] as [&OsStr; 2]);
} else { } else {
args.push("-an") args.push("-an".as_ref())
} }
} else { } else {
args.extend(["-c:a", "copy"]); args.extend(["-c:a".as_ref(), "copy".as_ref()] as [&OsStr; 2]);
} }
args.extend(["-f", output_format.ffmpeg_format(), output_path]); args.extend([
"-f".as_ref(),
output_format.ffmpeg_format().as_ref(),
output_path,
]);
Process::run("ffmpeg", &args, timeout)?.wait().await?; Process::run("ffmpeg", &args, &[], timeout)?.wait().await?;
Ok(()) Ok(())
} }

29
src/validate/magick.rs
View file

@ -1,8 +1,10 @@
use std::ffi::OsStr;
use actix_web::web::Bytes; use actix_web::web::Bytes;
use crate::{ use crate::{
formats::{AnimationFormat, ImageFormat}, formats::{AnimationFormat, ImageFormat},
magick::MagickError, magick::{MagickError, MAGICK_TEMPORARY_PATH},
process::Process, process::Process,
read::BoxRead, read::BoxRead,
tmp_file::TmpDir, tmp_file::TmpDir,
@ -57,8 +59,12 @@ async fn convert(
timeout: u64, timeout: u64,
bytes: Bytes, bytes: Bytes,
) -> Result<BoxRead<'static>, MagickError> { ) -> Result<BoxRead<'static>, MagickError> {
let temporary_path = tmp_dir
.tmp_folder()
.await
.map_err(MagickError::CreateTemporaryDirectory)?;
let input_file = tmp_dir.tmp_file(None); let input_file = tmp_dir.tmp_file(None);
let input_file_str = input_file.to_str().ok_or(MagickError::Path)?;
crate::store::file_store::safe_create_parent(&input_file) crate::store::file_store::safe_create_parent(&input_file)
.await .await
@ -73,27 +79,30 @@ async fn convert(
.map_err(MagickError::Write)?; .map_err(MagickError::Write)?;
tmp_one.close().await.map_err(MagickError::CloseFile)?; tmp_one.close().await.map_err(MagickError::CloseFile)?;
let input_arg = format!("{input}:{input_file_str}"); let input_arg = [input.as_ref(), input_file.as_os_str()].join(":".as_ref());
let output_arg = format!("{output}:-"); let output_arg = format!("{output}:-");
let quality = quality.map(|q| q.to_string()); let quality = quality.map(|q| q.to_string());
let mut args = vec!["convert"]; let mut args: Vec<&OsStr> = vec!["convert".as_ref()];
if coalesce { if coalesce {
args.push("-coalesce"); args.push("-coalesce".as_ref());
} }
args.extend(["-strip", "-auto-orient", &input_arg]); args.extend(["-strip".as_ref(), "-auto-orient".as_ref(), &input_arg] as [&OsStr; 3]);
if let Some(quality) = &quality { if let Some(quality) = &quality {
args.extend(["-quality", quality]); args.extend(["-quality".as_ref(), quality.as_ref()] as [&OsStr; 2]);
} }
args.push(&output_arg); args.push(output_arg.as_ref());
let reader = Process::run("magick", &args, timeout)?.read(); let envs = [(MAGICK_TEMPORARY_PATH, temporary_path.as_os_str())];
let clean_reader = crate::tmp_file::cleanup_tmpfile(reader, input_file); let reader = Process::run("magick", &args, &envs, timeout)?.read();
let clean_reader = input_file.reader(reader);
let clean_reader = temporary_path.reader(clean_reader);
Ok(Box::pin(clean_reader)) Ok(Box::pin(clean_reader))
} }