mirror of
https://git.asonix.dog/asonix/pict-rs
synced 2024-12-22 19:31:35 +00:00
Numerous changes:
- Improve error printing (display chain in addition to spantrace) - Fix migration (read main identifier from identifier tree, not filename tree) - Ensure uniqueness for processed images in ConcurrentProcessor (use source identifier in addition to thumbnail path, include extension in thumbnail path) - Update default log levels (make pict-rs quieter) - Add timeout for serving images from object storage (5 seconds)
This commit is contained in:
parent
3792a8923a
commit
1662f153ce
9 changed files with 195 additions and 64 deletions
|
@ -1,17 +1,42 @@
|
||||||
path = 'data/'
|
[server]
|
||||||
addr = '0.0.0.0:8080'
|
address = '0.0.0.0:8080'
|
||||||
|
[tracing.logging]
|
||||||
|
format = 'normal'
|
||||||
|
targets = 'info'
|
||||||
|
|
||||||
repo = 'sled'
|
[tracing.console]
|
||||||
store = 'object_storage'
|
buffer_capacity = 102400
|
||||||
|
|
||||||
[sled]
|
[tracing.opentelemetry]
|
||||||
sled_cache_capacity = 67108864
|
service_name = 'pict-rs'
|
||||||
|
targets = 'info'
|
||||||
|
|
||||||
[filesystem_storage]
|
[old_db]
|
||||||
filesystem_storage_path = '/mnt/files'
|
path = '/mnt'
|
||||||
|
|
||||||
[object_storage]
|
[media]
|
||||||
object_store_bucket_name = 'pict-rs'
|
max_width = 10000
|
||||||
object_store_region = 'http://minio:9000'
|
max_height = 10000
|
||||||
object_store_access_key = 'XZEZ5B8Y3UCINU1KCVF6'
|
max_area = 40000000
|
||||||
object_store_secret_key = 'cWbE5LcCK9YH8j1NvhOZocl+vH+b6T5Zvy3z+BZu'
|
max_file_size = 40
|
||||||
|
enable_silent_video = true
|
||||||
|
filters = [
|
||||||
|
'crop',
|
||||||
|
'resize',
|
||||||
|
'thumbnail',
|
||||||
|
'blur',
|
||||||
|
'identity',
|
||||||
|
]
|
||||||
|
skip_validate_imports = false
|
||||||
|
|
||||||
|
[repo]
|
||||||
|
type = 'sled'
|
||||||
|
path = '/mnt/sled-repo'
|
||||||
|
cache_capacity = 67108864
|
||||||
|
|
||||||
|
[store]
|
||||||
|
type = 'object_storage'
|
||||||
|
bucket_name = 'pict-rs'
|
||||||
|
region = 'http://minio:9000'
|
||||||
|
access_key = 'XZEZ5B8Y3UCINU1KCVF6'
|
||||||
|
secret_key = 'cWbE5LcCK9YH8j1NvhOZocl+vH+b6T5Zvy3z+BZu'
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
use crate::{
|
use crate::{
|
||||||
details::Details,
|
details::Details,
|
||||||
error::{Error, UploadError},
|
error::{Error, UploadError},
|
||||||
|
store::Identifier,
|
||||||
};
|
};
|
||||||
use actix_web::web;
|
use actix_web::web;
|
||||||
use dashmap::{mapref::entry::Entry, DashMap};
|
use dashmap::{mapref::entry::Entry, DashMap};
|
||||||
|
@ -16,13 +17,15 @@ use tracing::Span;
|
||||||
|
|
||||||
type OutcomeSender = Sender<(Details, web::Bytes)>;
|
type OutcomeSender = Sender<(Details, web::Bytes)>;
|
||||||
|
|
||||||
type ProcessMap = DashMap<PathBuf, Vec<OutcomeSender>>;
|
type ProcessMapKey = (Vec<u8>, PathBuf);
|
||||||
|
|
||||||
|
type ProcessMap = DashMap<ProcessMapKey, Vec<OutcomeSender>>;
|
||||||
|
|
||||||
static PROCESS_MAP: Lazy<ProcessMap> = Lazy::new(DashMap::new);
|
static PROCESS_MAP: Lazy<ProcessMap> = Lazy::new(DashMap::new);
|
||||||
|
|
||||||
struct CancelToken {
|
struct CancelToken {
|
||||||
span: Span,
|
span: Span,
|
||||||
path: PathBuf,
|
key: ProcessMapKey,
|
||||||
receiver: Option<Receiver<(Details, web::Bytes)>>,
|
receiver: Option<Receiver<(Details, web::Bytes)>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -39,14 +42,19 @@ impl<F> CancelSafeProcessor<F>
|
||||||
where
|
where
|
||||||
F: Future<Output = Result<(Details, web::Bytes), Error>>,
|
F: Future<Output = Result<(Details, web::Bytes), Error>>,
|
||||||
{
|
{
|
||||||
pub(super) fn new(path: PathBuf, fut: F) -> Self {
|
pub(super) fn new<I: Identifier>(identifier: I, path: PathBuf, fut: F) -> Result<Self, Error> {
|
||||||
let entry = PROCESS_MAP.entry(path.clone());
|
let id_bytes = identifier.to_bytes()?;
|
||||||
|
|
||||||
|
let key = (id_bytes, path.clone());
|
||||||
|
|
||||||
|
let entry = PROCESS_MAP.entry(key.clone());
|
||||||
|
|
||||||
let (receiver, span) = match entry {
|
let (receiver, span) = match entry {
|
||||||
Entry::Vacant(vacant) => {
|
Entry::Vacant(vacant) => {
|
||||||
vacant.insert(Vec::new());
|
vacant.insert(Vec::new());
|
||||||
let span = tracing::info_span!(
|
let span = tracing::info_span!(
|
||||||
"Processing image",
|
"Processing image",
|
||||||
|
identifier = &tracing::field::debug(&identifier),
|
||||||
path = &tracing::field::debug(&path),
|
path = &tracing::field::debug(&path),
|
||||||
completed = &tracing::field::Empty,
|
completed = &tracing::field::Empty,
|
||||||
);
|
);
|
||||||
|
@ -57,20 +65,21 @@ where
|
||||||
occupied.get_mut().push(tx);
|
occupied.get_mut().push(tx);
|
||||||
let span = tracing::info_span!(
|
let span = tracing::info_span!(
|
||||||
"Waiting for processed image",
|
"Waiting for processed image",
|
||||||
|
identifier = &tracing::field::debug(&identifier),
|
||||||
path = &tracing::field::debug(&path),
|
path = &tracing::field::debug(&path),
|
||||||
);
|
);
|
||||||
(Some(rx), span)
|
(Some(rx), span)
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
CancelSafeProcessor {
|
Ok(CancelSafeProcessor {
|
||||||
cancel_token: CancelToken {
|
cancel_token: CancelToken {
|
||||||
span,
|
span,
|
||||||
path,
|
key,
|
||||||
receiver,
|
receiver,
|
||||||
},
|
},
|
||||||
fut,
|
fut,
|
||||||
}
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -85,7 +94,7 @@ where
|
||||||
|
|
||||||
let span = &this.cancel_token.span;
|
let span = &this.cancel_token.span;
|
||||||
let receiver = &mut this.cancel_token.receiver;
|
let receiver = &mut this.cancel_token.receiver;
|
||||||
let path = &this.cancel_token.path;
|
let key = &this.cancel_token.key;
|
||||||
let fut = this.fut;
|
let fut = this.fut;
|
||||||
|
|
||||||
span.in_scope(|| {
|
span.in_scope(|| {
|
||||||
|
@ -95,7 +104,7 @@ where
|
||||||
.map(|res| res.map_err(|_| UploadError::Canceled.into()))
|
.map(|res| res.map_err(|_| UploadError::Canceled.into()))
|
||||||
} else {
|
} else {
|
||||||
fut.poll(cx).map(|res| {
|
fut.poll(cx).map(|res| {
|
||||||
let opt = PROCESS_MAP.remove(path);
|
let opt = PROCESS_MAP.remove(key);
|
||||||
res.map(|tup| {
|
res.map(|tup| {
|
||||||
if let Some((_, vec)) = opt {
|
if let Some((_, vec)) = opt {
|
||||||
for sender in vec {
|
for sender in vec {
|
||||||
|
@ -113,7 +122,7 @@ where
|
||||||
impl Drop for CancelToken {
|
impl Drop for CancelToken {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
if self.receiver.is_none() {
|
if self.receiver.is_none() {
|
||||||
let completed = PROCESS_MAP.remove(&self.path).is_none();
|
let completed = PROCESS_MAP.remove(&self.key).is_none();
|
||||||
self.span.record("completed", &completed);
|
self.span.record("completed", &completed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -108,7 +108,9 @@ impl Default for LoggingDefaults {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
LoggingDefaults {
|
LoggingDefaults {
|
||||||
format: LogFormat::Normal,
|
format: LogFormat::Normal,
|
||||||
targets: "info".parse().expect("Valid targets string"),
|
targets: "warn,tracing_actix_web=info,actix_web=info,actix_server=info"
|
||||||
|
.parse()
|
||||||
|
.expect("Valid targets string"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -182,13 +182,13 @@ impl FromStr for ImageFormat {
|
||||||
type Err = String;
|
type Err = String;
|
||||||
|
|
||||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||||
for variant in Self::value_variants() {
|
match s.to_lowercase().as_str() {
|
||||||
if variant.to_possible_value().unwrap().matches(s, false) {
|
"jpeg" | "jpg" => Ok(Self::Jpeg),
|
||||||
return Ok(*variant);
|
"png" => Ok(Self::Png),
|
||||||
|
"webp" => Ok(Self::Webp),
|
||||||
|
other => Err(format!("Invalid variant: {}", other)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(format!("Invalid variant: {}", s))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl FromStr for LogFormat {
|
impl FromStr for LogFormat {
|
||||||
|
|
37
src/error.rs
37
src/error.rs
|
@ -16,23 +16,32 @@ impl std::fmt::Display for Error {
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
writeln!(f, "{}", self.kind)?;
|
writeln!(f, "{}", self.kind)?;
|
||||||
writeln!(f)?;
|
writeln!(f)?;
|
||||||
let mut count = 0;
|
|
||||||
let mut source = std::error::Error::source(self);
|
|
||||||
if source.is_some() {
|
|
||||||
writeln!(f, "Chain:")?;
|
writeln!(f, "Chain:")?;
|
||||||
}
|
fmt_chain(f, &self.kind)?;
|
||||||
while let Some(err) = source {
|
|
||||||
write!(f, "{}. ", count)?;
|
|
||||||
writeln!(f, "{}", err)?;
|
|
||||||
|
|
||||||
count += 1;
|
|
||||||
source = std::error::Error::source(err);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
writeln!(f)?;
|
||||||
|
writeln!(f, "Spantrace:")?;
|
||||||
std::fmt::Display::fmt(&self.context, f)
|
std::fmt::Display::fmt(&self.context, f)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn fmt_chain(
|
||||||
|
f: &mut std::fmt::Formatter<'_>,
|
||||||
|
err: &dyn std::error::Error,
|
||||||
|
) -> Result<usize, std::fmt::Error> {
|
||||||
|
let count = if let Some(source) = std::error::Error::source(err) {
|
||||||
|
fmt_chain(f, source)?
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
|
|
||||||
|
write!(f, "\t{}. ", count)?;
|
||||||
|
writeln!(f, "{}", err)?;
|
||||||
|
|
||||||
|
Ok(count + 1)
|
||||||
|
}
|
||||||
|
|
||||||
impl std::error::Error for Error {
|
impl std::error::Error for Error {
|
||||||
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
|
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
|
||||||
self.kind.source()
|
self.kind.source()
|
||||||
|
@ -113,9 +122,6 @@ pub(crate) enum UploadError {
|
||||||
#[error("Unable to send request, {0}")]
|
#[error("Unable to send request, {0}")]
|
||||||
SendRequest(String),
|
SendRequest(String),
|
||||||
|
|
||||||
#[error("No filename provided in request")]
|
|
||||||
MissingFilename,
|
|
||||||
|
|
||||||
#[error("Error converting Path to String")]
|
#[error("Error converting Path to String")]
|
||||||
Path,
|
Path,
|
||||||
|
|
||||||
|
@ -157,7 +163,8 @@ impl ResponseError for Error {
|
||||||
| UploadError::Limit(_)
|
| UploadError::Limit(_)
|
||||||
| UploadError::NoFiles
|
| UploadError::NoFiles
|
||||||
| UploadError::Upload(_) => StatusCode::BAD_REQUEST,
|
| UploadError::Upload(_) => StatusCode::BAD_REQUEST,
|
||||||
UploadError::MissingAlias | UploadError::MissingFilename => StatusCode::NOT_FOUND,
|
UploadError::Sled(crate::repo::sled::SledError::Missing)
|
||||||
|
| UploadError::MissingAlias => StatusCode::NOT_FOUND,
|
||||||
UploadError::InvalidToken => StatusCode::FORBIDDEN,
|
UploadError::InvalidToken => StatusCode::FORBIDDEN,
|
||||||
UploadError::Range => StatusCode::RANGE_NOT_SATISFIABLE,
|
UploadError::Range => StatusCode::RANGE_NOT_SATISFIABLE,
|
||||||
_ => StatusCode::INTERNAL_SERVER_ERROR,
|
_ => StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
|
26
src/main.rs
26
src/main.rs
|
@ -263,7 +263,7 @@ type ProcessQuery = Vec<(String, String)>;
|
||||||
fn prepare_process(
|
fn prepare_process(
|
||||||
query: web::Query<ProcessQuery>,
|
query: web::Query<ProcessQuery>,
|
||||||
ext: &str,
|
ext: &str,
|
||||||
filters: &Option<HashSet<String>>,
|
filters: &HashSet<String>,
|
||||||
) -> Result<(ImageFormat, Alias, PathBuf, Vec<String>), Error> {
|
) -> Result<(ImageFormat, Alias, PathBuf, Vec<String>), Error> {
|
||||||
let (alias, operations) =
|
let (alias, operations) =
|
||||||
query
|
query
|
||||||
|
@ -279,25 +279,23 @@ fn prepare_process(
|
||||||
});
|
});
|
||||||
|
|
||||||
if alias.is_empty() {
|
if alias.is_empty() {
|
||||||
return Err(UploadError::MissingFilename.into());
|
return Err(UploadError::MissingAlias.into());
|
||||||
}
|
}
|
||||||
|
|
||||||
let alias = Alias::from_existing(&alias);
|
let alias = Alias::from_existing(&alias);
|
||||||
|
|
||||||
let operations = if let Some(filters) = filters.as_ref() {
|
let operations = operations
|
||||||
operations
|
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter(|(k, _)| filters.contains(&k.to_lowercase()))
|
.filter(|(k, _)| filters.contains(&k.to_lowercase()))
|
||||||
.collect()
|
.collect::<Vec<_>>();
|
||||||
} else {
|
|
||||||
operations
|
|
||||||
};
|
|
||||||
|
|
||||||
let format = ext
|
let format = ext
|
||||||
.parse::<ImageFormat>()
|
.parse::<ImageFormat>()
|
||||||
.map_err(|_| UploadError::UnsupportedFormat)?;
|
.map_err(|_| UploadError::UnsupportedFormat)?;
|
||||||
|
|
||||||
let (thumbnail_path, thumbnail_args) = self::processor::build_chain(&operations)?;
|
let ext = format.to_string();
|
||||||
|
|
||||||
|
let (thumbnail_path, thumbnail_args) = self::processor::build_chain(&operations, &ext)?;
|
||||||
|
|
||||||
Ok((format, alias, thumbnail_path, thumbnail_args))
|
Ok((format, alias, thumbnail_path, thumbnail_args))
|
||||||
}
|
}
|
||||||
|
@ -308,7 +306,7 @@ async fn process_details<S: Store>(
|
||||||
ext: web::Path<String>,
|
ext: web::Path<String>,
|
||||||
manager: web::Data<UploadManager>,
|
manager: web::Data<UploadManager>,
|
||||||
store: web::Data<S>,
|
store: web::Data<S>,
|
||||||
filters: web::Data<Option<HashSet<String>>>,
|
filters: web::Data<HashSet<String>>,
|
||||||
) -> Result<HttpResponse, Error> {
|
) -> Result<HttpResponse, Error> {
|
||||||
let (_, alias, thumbnail_path, _) = prepare_process(query, ext.as_str(), &filters)?;
|
let (_, alias, thumbnail_path, _) = prepare_process(query, ext.as_str(), &filters)?;
|
||||||
|
|
||||||
|
@ -332,7 +330,7 @@ async fn process<S: Store + 'static>(
|
||||||
ext: web::Path<String>,
|
ext: web::Path<String>,
|
||||||
manager: web::Data<UploadManager>,
|
manager: web::Data<UploadManager>,
|
||||||
store: web::Data<S>,
|
store: web::Data<S>,
|
||||||
filters: web::Data<Option<HashSet<String>>>,
|
filters: web::Data<HashSet<String>>,
|
||||||
) -> Result<HttpResponse, Error> {
|
) -> Result<HttpResponse, Error> {
|
||||||
let (format, alias, thumbnail_path, thumbnail_args) =
|
let (format, alias, thumbnail_path, thumbnail_args) =
|
||||||
prepare_process(query, ext.as_str(), &filters)?;
|
prepare_process(query, ext.as_str(), &filters)?;
|
||||||
|
@ -361,6 +359,7 @@ async fn process<S: Store + 'static>(
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let thumbnail_path2 = thumbnail_path.clone();
|
let thumbnail_path2 = thumbnail_path.clone();
|
||||||
|
let identifier2 = identifier.clone();
|
||||||
let process_fut = async {
|
let process_fut = async {
|
||||||
let thumbnail_path = thumbnail_path2;
|
let thumbnail_path = thumbnail_path2;
|
||||||
|
|
||||||
|
@ -368,7 +367,7 @@ async fn process<S: Store + 'static>(
|
||||||
|
|
||||||
let mut processed_reader = crate::magick::process_image_store_read(
|
let mut processed_reader = crate::magick::process_image_store_read(
|
||||||
(**store).clone(),
|
(**store).clone(),
|
||||||
identifier,
|
identifier2,
|
||||||
thumbnail_args,
|
thumbnail_args,
|
||||||
format,
|
format,
|
||||||
)?;
|
)?;
|
||||||
|
@ -417,7 +416,8 @@ async fn process<S: Store + 'static>(
|
||||||
Ok((details, bytes)) as Result<(Details, web::Bytes), Error>
|
Ok((details, bytes)) as Result<(Details, web::Bytes), Error>
|
||||||
};
|
};
|
||||||
|
|
||||||
let (details, bytes) = CancelSafeProcessor::new(thumbnail_path.clone(), process_fut).await?;
|
let (details, bytes) =
|
||||||
|
CancelSafeProcessor::new(identifier, thumbnail_path.clone(), process_fut)?.await?;
|
||||||
|
|
||||||
let (builder, stream) = if let Some(web::Header(range_header)) = range {
|
let (builder, stream) = if let Some(web::Header(range_header)) = range {
|
||||||
if let Some(range) = range::single_bytes_range(&range_header) {
|
if let Some(range) = range::single_bytes_range(&range_header) {
|
||||||
|
|
|
@ -20,7 +20,10 @@ pub(crate) struct Crop(usize, usize);
|
||||||
pub(crate) struct Blur(f64);
|
pub(crate) struct Blur(f64);
|
||||||
|
|
||||||
#[instrument]
|
#[instrument]
|
||||||
pub(crate) fn build_chain(args: &[(String, String)]) -> Result<(PathBuf, Vec<String>), Error> {
|
pub(crate) fn build_chain(
|
||||||
|
args: &[(String, String)],
|
||||||
|
ext: &str,
|
||||||
|
) -> Result<(PathBuf, Vec<String>), Error> {
|
||||||
fn parse<P: Processor>(key: &str, value: &str) -> Result<Option<P>, UploadError> {
|
fn parse<P: Processor>(key: &str, value: &str) -> Result<Option<P>, UploadError> {
|
||||||
if key == P::NAME {
|
if key == P::NAME {
|
||||||
return Ok(Some(P::parse(key, value).ok_or(UploadError::ParsePath)?));
|
return Ok(Some(P::parse(key, value).ok_or(UploadError::ParsePath)?));
|
||||||
|
@ -37,7 +40,7 @@ pub(crate) fn build_chain(args: &[(String, String)]) -> Result<(PathBuf, Vec<Str
|
||||||
}};
|
}};
|
||||||
}
|
}
|
||||||
|
|
||||||
let (path, args) =
|
let (mut path, args) =
|
||||||
args.iter()
|
args.iter()
|
||||||
.fold(Ok((PathBuf::default(), vec![])), |inner, (name, value)| {
|
.fold(Ok((PathBuf::default(), vec![])), |inner, (name, value)| {
|
||||||
if let Ok(inner) = inner {
|
if let Ok(inner) = inner {
|
||||||
|
@ -53,6 +56,8 @@ pub(crate) fn build_chain(args: &[(String, String)]) -> Result<(PathBuf, Vec<Str
|
||||||
}
|
}
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
|
path.push(ext);
|
||||||
|
|
||||||
Ok((path, args))
|
Ok((path, args))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -87,7 +87,7 @@ impl Old {
|
||||||
.get(hash)?
|
.get(hash)?
|
||||||
.ok_or_else(|| anyhow::anyhow!("Missing filename"))?;
|
.ok_or_else(|| anyhow::anyhow!("Missing filename"))?;
|
||||||
|
|
||||||
self.filename_tree
|
self.identifier_tree
|
||||||
.get(filename)?
|
.get(filename)?
|
||||||
.ok_or_else(|| anyhow::anyhow!("Missing identifier"))
|
.ok_or_else(|| anyhow::anyhow!("Missing identifier"))
|
||||||
}
|
}
|
||||||
|
@ -114,7 +114,9 @@ impl Old {
|
||||||
|
|
||||||
let path = String::from_utf8(variant_path_bytes.to_vec()).ok()?;
|
let path = String::from_utf8(variant_path_bytes.to_vec()).ok()?;
|
||||||
let mut path = PathBuf::from(path);
|
let mut path = PathBuf::from(path);
|
||||||
|
let extension = path.extension()?.to_str()?.to_string();
|
||||||
path.pop();
|
path.pop();
|
||||||
|
path.push(extension);
|
||||||
|
|
||||||
Some((path, value))
|
Some((path, value))
|
||||||
})
|
})
|
||||||
|
|
|
@ -3,15 +3,21 @@ use crate::{
|
||||||
repo::{Repo, SettingsRepo},
|
repo::{Repo, SettingsRepo},
|
||||||
store::Store,
|
store::Store,
|
||||||
};
|
};
|
||||||
|
use actix_rt::time::Sleep;
|
||||||
use actix_web::web::Bytes;
|
use actix_web::web::Bytes;
|
||||||
use futures_util::stream::Stream;
|
use futures_util::stream::Stream;
|
||||||
use s3::{
|
use s3::{
|
||||||
client::Client, command::Command, creds::Credentials, request_trait::Request, Bucket, Region,
|
client::Client, command::Command, creds::Credentials, request_trait::Request, Bucket, Region,
|
||||||
};
|
};
|
||||||
use std::{
|
use std::{
|
||||||
|
future::Future,
|
||||||
pin::Pin,
|
pin::Pin,
|
||||||
string::FromUtf8Error,
|
string::FromUtf8Error,
|
||||||
task::{Context, Poll},
|
sync::{
|
||||||
|
atomic::{AtomicBool, Ordering},
|
||||||
|
Arc,
|
||||||
|
},
|
||||||
|
task::{Context, Poll, Wake, Waker},
|
||||||
};
|
};
|
||||||
use storage_path_generator::{Generator, Path};
|
use storage_path_generator::{Generator, Path};
|
||||||
use tokio::io::{AsyncRead, AsyncWrite};
|
use tokio::io::{AsyncRead, AsyncWrite};
|
||||||
|
@ -54,6 +60,17 @@ pin_project_lite::pin_project! {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pin_project_lite::pin_project! {
|
||||||
|
struct Timeout<S> {
|
||||||
|
sleep: Option<Pin<Box<Sleep>>>,
|
||||||
|
|
||||||
|
woken: Arc<AtomicBool>,
|
||||||
|
|
||||||
|
#[pin]
|
||||||
|
inner: S,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[async_trait::async_trait(?Send)]
|
#[async_trait::async_trait(?Send)]
|
||||||
impl Store for ObjectStore {
|
impl Store for ObjectStore {
|
||||||
type Identifier = ObjectId;
|
type Identifier = ObjectId;
|
||||||
|
@ -107,7 +124,10 @@ impl Store for ObjectStore {
|
||||||
|
|
||||||
let response = request.response().await.map_err(ObjectError::from)?;
|
let response = request.response().await.map_err(ObjectError::from)?;
|
||||||
|
|
||||||
Ok(Box::pin(io_error(response.bytes_stream())))
|
Ok(Box::pin(timeout(
|
||||||
|
io_error(response.bytes_stream()),
|
||||||
|
std::time::Duration::from_secs(5),
|
||||||
|
)))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tracing::instrument(skip(writer))]
|
#[tracing::instrument(skip(writer))]
|
||||||
|
@ -238,6 +258,17 @@ where
|
||||||
IoError { inner: stream }
|
IoError { inner: stream }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn timeout<S, T>(stream: S, duration: std::time::Duration) -> impl Stream<Item = std::io::Result<T>>
|
||||||
|
where
|
||||||
|
S: Stream<Item = std::io::Result<T>>,
|
||||||
|
{
|
||||||
|
Timeout {
|
||||||
|
sleep: Some(Box::pin(actix_rt::time::sleep(duration))),
|
||||||
|
woken: Arc::new(AtomicBool::new(true)),
|
||||||
|
inner: stream,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl<S, T, E> Stream for IoError<S>
|
impl<S, T, E> Stream for IoError<S>
|
||||||
where
|
where
|
||||||
S: Stream<Item = Result<T, E>>,
|
S: Stream<Item = Result<T, E>>,
|
||||||
|
@ -254,6 +285,56 @@ where
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct TimeoutWaker {
|
||||||
|
woken: Arc<AtomicBool>,
|
||||||
|
inner: Waker,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Wake for TimeoutWaker {
|
||||||
|
fn wake(self: Arc<Self>) {
|
||||||
|
self.wake_by_ref()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn wake_by_ref(self: &Arc<Self>) {
|
||||||
|
self.woken.store(true, Ordering::Release);
|
||||||
|
self.inner.wake_by_ref();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<S, T> Stream for Timeout<S>
|
||||||
|
where
|
||||||
|
S: Stream<Item = std::io::Result<T>>,
|
||||||
|
{
|
||||||
|
type Item = std::io::Result<T>;
|
||||||
|
|
||||||
|
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||||
|
let this = self.as_mut().project();
|
||||||
|
|
||||||
|
if this.woken.swap(false, Ordering::Acquire) {
|
||||||
|
if let Some(mut sleep) = this.sleep.take() {
|
||||||
|
let timeout_waker = Arc::new(TimeoutWaker {
|
||||||
|
woken: Arc::clone(this.woken),
|
||||||
|
inner: cx.waker().clone(),
|
||||||
|
})
|
||||||
|
.into();
|
||||||
|
let mut timeout_cx = Context::from_waker(&timeout_waker);
|
||||||
|
if let Poll::Ready(()) = sleep.as_mut().poll(&mut timeout_cx) {
|
||||||
|
return Poll::Ready(Some(Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::Other,
|
||||||
|
"Stream timeout".to_string(),
|
||||||
|
))));
|
||||||
|
} else {
|
||||||
|
*this.sleep = Some(sleep);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return Poll::Ready(None);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.inner.poll_next(cx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl std::fmt::Debug for ObjectStore {
|
impl std::fmt::Debug for ObjectStore {
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
f.debug_struct("ObjectStore")
|
f.debug_struct("ObjectStore")
|
||||||
|
|
Loading…
Reference in a new issue