2021-10-23 04:48:56 +00:00
|
|
|
use actix_web::web::Bytes;
|
2023-08-23 16:59:42 +00:00
|
|
|
use futures_core::Stream;
|
2023-08-16 00:19:03 +00:00
|
|
|
use std::{fmt::Debug, sync::Arc};
|
2023-10-03 22:04:40 +00:00
|
|
|
use streem::IntoStreamer;
|
2021-10-23 04:48:56 +00:00
|
|
|
use tokio::io::{AsyncRead, AsyncWrite};
|
|
|
|
|
2023-10-03 22:04:40 +00:00
|
|
|
use crate::{bytes_stream::BytesStream, error_code::ErrorCode, stream::LocalBoxStream};
|
2023-09-02 01:50:10 +00:00
|
|
|
|
2021-10-23 04:48:56 +00:00
|
|
|
pub(crate) mod file_store;
|
2021-10-28 04:06:03 +00:00
|
|
|
pub(crate) mod object_store;
|
2021-10-23 04:48:56 +00:00
|
|
|
|
2023-06-20 20:59:08 +00:00
|
|
|
#[derive(Debug, thiserror::Error)]
|
|
|
|
pub(crate) enum StoreError {
|
|
|
|
#[error("Error in file store")]
|
|
|
|
FileStore(#[source] crate::store::file_store::FileError),
|
|
|
|
|
|
|
|
#[error("Error in object store")]
|
|
|
|
ObjectStore(#[source] crate::store::object_store::ObjectError),
|
|
|
|
|
|
|
|
#[error("Error in DB")]
|
|
|
|
Repo(#[from] crate::repo::RepoError),
|
|
|
|
|
2023-08-13 19:12:38 +00:00
|
|
|
#[error("Error in 0.4 DB")]
|
|
|
|
Repo04(#[from] crate::repo_04::RepoError),
|
|
|
|
|
2023-10-03 22:04:40 +00:00
|
|
|
#[error("Error reading bytes stream")]
|
|
|
|
ReadStream(#[source] std::io::Error),
|
|
|
|
|
2023-06-20 20:59:08 +00:00
|
|
|
#[error("Requested file is not found")]
|
2023-07-03 02:07:15 +00:00
|
|
|
FileNotFound(#[source] std::io::Error),
|
|
|
|
|
|
|
|
#[error("Requested object is not found")]
|
|
|
|
ObjectNotFound(#[source] crate::store::object_store::ObjectError),
|
2023-06-20 20:59:08 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
impl StoreError {
|
2023-09-02 01:50:10 +00:00
|
|
|
pub(crate) const fn error_code(&self) -> ErrorCode {
|
|
|
|
match self {
|
|
|
|
Self::FileStore(e) => e.error_code(),
|
|
|
|
Self::ObjectStore(e) => e.error_code(),
|
|
|
|
Self::Repo(e) => e.error_code(),
|
2023-10-03 22:04:40 +00:00
|
|
|
Self::ReadStream(_) => ErrorCode::IO_ERROR,
|
2023-09-02 01:50:10 +00:00
|
|
|
Self::Repo04(_) => ErrorCode::OLD_REPO_ERROR,
|
|
|
|
Self::FileNotFound(_) | Self::ObjectNotFound(_) => ErrorCode::NOT_FOUND,
|
|
|
|
}
|
|
|
|
}
|
2023-09-05 02:51:27 +00:00
|
|
|
|
2023-06-20 20:59:08 +00:00
|
|
|
pub(crate) const fn is_not_found(&self) -> bool {
|
2023-07-03 02:07:15 +00:00
|
|
|
matches!(self, Self::FileNotFound(_)) || matches!(self, Self::ObjectNotFound(_))
|
2023-06-20 20:59:08 +00:00
|
|
|
}
|
2023-09-05 02:51:27 +00:00
|
|
|
|
|
|
|
pub(crate) const fn is_disconnected(&self) -> bool {
|
|
|
|
match self {
|
|
|
|
Self::Repo(e) => e.is_disconnected(),
|
|
|
|
_ => false,
|
|
|
|
}
|
|
|
|
}
|
2023-06-20 20:59:08 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
impl From<crate::store::file_store::FileError> for StoreError {
|
|
|
|
fn from(value: crate::store::file_store::FileError) -> Self {
|
|
|
|
match value {
|
|
|
|
crate::store::file_store::FileError::Io(e)
|
|
|
|
if e.kind() == std::io::ErrorKind::NotFound =>
|
|
|
|
{
|
2023-07-03 02:07:15 +00:00
|
|
|
Self::FileNotFound(e)
|
2023-06-20 20:59:08 +00:00
|
|
|
}
|
|
|
|
e => Self::FileStore(e),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl From<crate::store::object_store::ObjectError> for StoreError {
|
|
|
|
fn from(value: crate::store::object_store::ObjectError) -> Self {
|
|
|
|
match value {
|
2023-07-03 02:07:15 +00:00
|
|
|
e @ crate::store::object_store::ObjectError::Status(
|
2023-06-20 20:59:08 +00:00
|
|
|
actix_web::http::StatusCode::NOT_FOUND,
|
|
|
|
_,
|
2023-12-11 19:05:29 +00:00
|
|
|
_,
|
2023-07-03 02:07:15 +00:00
|
|
|
) => Self::ObjectNotFound(e),
|
2023-06-20 20:59:08 +00:00
|
|
|
e => Self::ObjectStore(e),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-09-24 19:18:49 +00:00
|
|
|
pub(crate) trait Store: Clone + Debug {
|
2023-07-07 17:05:42 +00:00
|
|
|
async fn health_check(&self) -> Result<(), StoreError>;
|
|
|
|
|
2023-07-14 19:53:37 +00:00
|
|
|
async fn save_async_read<Reader>(
|
|
|
|
&self,
|
|
|
|
reader: Reader,
|
|
|
|
content_type: mime::Mime,
|
2023-09-02 23:30:45 +00:00
|
|
|
) -> Result<Arc<str>, StoreError>
|
2022-09-24 22:18:53 +00:00
|
|
|
where
|
|
|
|
Reader: AsyncRead + Unpin + 'static;
|
2022-09-24 19:18:49 +00:00
|
|
|
|
2023-07-14 19:53:37 +00:00
|
|
|
async fn save_stream<S>(
|
|
|
|
&self,
|
|
|
|
stream: S,
|
|
|
|
content_type: mime::Mime,
|
2023-09-02 23:30:45 +00:00
|
|
|
) -> Result<Arc<str>, StoreError>
|
2021-10-23 04:48:56 +00:00
|
|
|
where
|
2022-09-24 22:18:53 +00:00
|
|
|
S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static;
|
2021-10-23 04:48:56 +00:00
|
|
|
|
2023-07-14 19:53:37 +00:00
|
|
|
async fn save_bytes(
|
|
|
|
&self,
|
|
|
|
bytes: Bytes,
|
|
|
|
content_type: mime::Mime,
|
2023-09-02 23:30:45 +00:00
|
|
|
) -> Result<Arc<str>, StoreError>;
|
2023-07-14 19:53:37 +00:00
|
|
|
|
2023-09-02 23:30:45 +00:00
|
|
|
fn public_url(&self, _: &Arc<str>) -> Option<url::Url>;
|
2021-10-23 04:48:56 +00:00
|
|
|
|
|
|
|
async fn to_stream(
|
|
|
|
&self,
|
2023-09-02 23:30:45 +00:00
|
|
|
identifier: &Arc<str>,
|
2021-10-23 04:48:56 +00:00
|
|
|
from_start: Option<u64>,
|
|
|
|
len: Option<u64>,
|
2023-09-02 23:30:45 +00:00
|
|
|
) -> Result<LocalBoxStream<'static, std::io::Result<Bytes>>, StoreError>;
|
2021-10-23 04:48:56 +00:00
|
|
|
|
2023-10-03 22:04:40 +00:00
|
|
|
async fn to_bytes(
|
|
|
|
&self,
|
|
|
|
identifier: &Arc<str>,
|
|
|
|
from_start: Option<u64>,
|
|
|
|
len: Option<u64>,
|
|
|
|
) -> Result<BytesStream, StoreError> {
|
|
|
|
let mut buf = BytesStream::new();
|
|
|
|
|
|
|
|
let mut streamer = self
|
|
|
|
.to_stream(identifier, from_start, len)
|
|
|
|
.await?
|
|
|
|
.into_streamer();
|
|
|
|
|
|
|
|
while let Some(bytes) = streamer.try_next().await.map_err(StoreError::ReadStream)? {
|
2023-12-28 17:58:38 +00:00
|
|
|
tracing::trace!("to_bytes: looping");
|
|
|
|
|
2023-10-03 22:04:40 +00:00
|
|
|
buf.add_bytes(bytes);
|
|
|
|
}
|
|
|
|
|
|
|
|
Ok(buf)
|
|
|
|
}
|
|
|
|
|
2021-10-23 04:48:56 +00:00
|
|
|
async fn read_into<Writer>(
|
|
|
|
&self,
|
2023-09-02 23:30:45 +00:00
|
|
|
identifier: &Arc<str>,
|
2021-10-23 04:48:56 +00:00
|
|
|
writer: &mut Writer,
|
|
|
|
) -> Result<(), std::io::Error>
|
|
|
|
where
|
2022-09-24 22:18:53 +00:00
|
|
|
Writer: AsyncWrite + Unpin;
|
2021-10-23 04:48:56 +00:00
|
|
|
|
2023-09-02 23:30:45 +00:00
|
|
|
async fn len(&self, identifier: &Arc<str>) -> Result<u64, StoreError>;
|
2021-10-23 04:48:56 +00:00
|
|
|
|
2023-09-02 23:30:45 +00:00
|
|
|
async fn remove(&self, identifier: &Arc<str>) -> Result<(), StoreError>;
|
2021-10-23 04:48:56 +00:00
|
|
|
}
|
2022-04-01 21:51:12 +00:00
|
|
|
|
2022-09-11 00:29:22 +00:00
|
|
|
impl<T> Store for actix_web::web::Data<T>
|
|
|
|
where
|
|
|
|
T: Store,
|
|
|
|
{
|
2023-07-07 17:05:42 +00:00
|
|
|
async fn health_check(&self) -> Result<(), StoreError> {
|
|
|
|
T::health_check(self).await
|
|
|
|
}
|
|
|
|
|
2023-07-14 19:53:37 +00:00
|
|
|
async fn save_async_read<Reader>(
|
|
|
|
&self,
|
|
|
|
reader: Reader,
|
|
|
|
content_type: mime::Mime,
|
2023-09-02 23:30:45 +00:00
|
|
|
) -> Result<Arc<str>, StoreError>
|
2022-09-11 00:29:22 +00:00
|
|
|
where
|
2022-09-24 22:18:53 +00:00
|
|
|
Reader: AsyncRead + Unpin + 'static,
|
2022-09-11 00:29:22 +00:00
|
|
|
{
|
2023-07-14 19:53:37 +00:00
|
|
|
T::save_async_read(self, reader, content_type).await
|
2022-09-11 00:29:22 +00:00
|
|
|
}
|
|
|
|
|
2023-07-14 19:53:37 +00:00
|
|
|
async fn save_stream<S>(
|
|
|
|
&self,
|
|
|
|
stream: S,
|
|
|
|
content_type: mime::Mime,
|
2023-09-02 23:30:45 +00:00
|
|
|
) -> Result<Arc<str>, StoreError>
|
2022-09-24 22:18:53 +00:00
|
|
|
where
|
|
|
|
S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static,
|
|
|
|
{
|
2023-07-14 19:53:37 +00:00
|
|
|
T::save_stream(self, stream, content_type).await
|
2022-09-24 22:18:53 +00:00
|
|
|
}
|
|
|
|
|
2023-07-14 19:53:37 +00:00
|
|
|
async fn save_bytes(
|
|
|
|
&self,
|
|
|
|
bytes: Bytes,
|
|
|
|
content_type: mime::Mime,
|
2023-09-02 23:30:45 +00:00
|
|
|
) -> Result<Arc<str>, StoreError> {
|
2023-07-14 19:53:37 +00:00
|
|
|
T::save_bytes(self, bytes, content_type).await
|
2022-09-11 00:29:22 +00:00
|
|
|
}
|
|
|
|
|
2023-09-02 23:30:45 +00:00
|
|
|
fn public_url(&self, identifier: &Arc<str>) -> Option<url::Url> {
|
2023-07-14 20:23:07 +00:00
|
|
|
T::public_url(self, identifier)
|
|
|
|
}
|
|
|
|
|
2022-09-11 00:29:22 +00:00
|
|
|
async fn to_stream(
|
|
|
|
&self,
|
2023-09-02 23:30:45 +00:00
|
|
|
identifier: &Arc<str>,
|
2022-09-11 00:29:22 +00:00
|
|
|
from_start: Option<u64>,
|
|
|
|
len: Option<u64>,
|
2023-09-02 23:30:45 +00:00
|
|
|
) -> Result<LocalBoxStream<'static, std::io::Result<Bytes>>, StoreError> {
|
2022-09-11 15:04:37 +00:00
|
|
|
T::to_stream(self, identifier, from_start, len).await
|
2022-09-11 00:29:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
async fn read_into<Writer>(
|
|
|
|
&self,
|
2023-09-02 23:30:45 +00:00
|
|
|
identifier: &Arc<str>,
|
2022-09-11 00:29:22 +00:00
|
|
|
writer: &mut Writer,
|
|
|
|
) -> Result<(), std::io::Error>
|
|
|
|
where
|
2022-09-24 22:18:53 +00:00
|
|
|
Writer: AsyncWrite + Unpin,
|
2022-09-11 00:29:22 +00:00
|
|
|
{
|
2022-09-11 15:04:37 +00:00
|
|
|
T::read_into(self, identifier, writer).await
|
2022-09-11 00:29:22 +00:00
|
|
|
}
|
|
|
|
|
2023-09-02 23:30:45 +00:00
|
|
|
async fn len(&self, identifier: &Arc<str>) -> Result<u64, StoreError> {
|
2022-09-11 15:04:37 +00:00
|
|
|
T::len(self, identifier).await
|
2022-09-11 00:29:22 +00:00
|
|
|
}
|
|
|
|
|
2023-09-02 23:30:45 +00:00
|
|
|
async fn remove(&self, identifier: &Arc<str>) -> Result<(), StoreError> {
|
2022-09-11 15:04:37 +00:00
|
|
|
T::remove(self, identifier).await
|
2022-09-11 00:29:22 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-09-02 23:30:45 +00:00
|
|
|
impl<T> Store for Arc<T>
|
2022-04-01 21:51:12 +00:00
|
|
|
where
|
|
|
|
T: Store,
|
|
|
|
{
|
2023-07-07 17:05:42 +00:00
|
|
|
async fn health_check(&self) -> Result<(), StoreError> {
|
|
|
|
T::health_check(self).await
|
|
|
|
}
|
|
|
|
|
2023-07-14 19:53:37 +00:00
|
|
|
async fn save_async_read<Reader>(
|
|
|
|
&self,
|
|
|
|
reader: Reader,
|
|
|
|
content_type: mime::Mime,
|
2023-09-02 23:30:45 +00:00
|
|
|
) -> Result<Arc<str>, StoreError>
|
2022-04-01 21:51:12 +00:00
|
|
|
where
|
2022-09-24 22:18:53 +00:00
|
|
|
Reader: AsyncRead + Unpin + 'static,
|
2022-04-01 21:51:12 +00:00
|
|
|
{
|
2023-07-14 19:53:37 +00:00
|
|
|
T::save_async_read(self, reader, content_type).await
|
2022-04-01 21:51:12 +00:00
|
|
|
}
|
|
|
|
|
2023-07-14 19:53:37 +00:00
|
|
|
async fn save_stream<S>(
|
|
|
|
&self,
|
|
|
|
stream: S,
|
|
|
|
content_type: mime::Mime,
|
2023-09-02 23:30:45 +00:00
|
|
|
) -> Result<Arc<str>, StoreError>
|
2022-09-24 22:18:53 +00:00
|
|
|
where
|
|
|
|
S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static,
|
|
|
|
{
|
2023-07-14 19:53:37 +00:00
|
|
|
T::save_stream(self, stream, content_type).await
|
2022-09-24 22:18:53 +00:00
|
|
|
}
|
|
|
|
|
2023-07-14 19:53:37 +00:00
|
|
|
async fn save_bytes(
|
|
|
|
&self,
|
|
|
|
bytes: Bytes,
|
|
|
|
content_type: mime::Mime,
|
2023-09-02 23:30:45 +00:00
|
|
|
) -> Result<Arc<str>, StoreError> {
|
2023-07-14 19:53:37 +00:00
|
|
|
T::save_bytes(self, bytes, content_type).await
|
2022-04-01 21:51:12 +00:00
|
|
|
}
|
|
|
|
|
2023-09-02 23:30:45 +00:00
|
|
|
fn public_url(&self, identifier: &Arc<str>) -> Option<url::Url> {
|
2023-07-14 20:23:07 +00:00
|
|
|
T::public_url(self, identifier)
|
|
|
|
}
|
|
|
|
|
2022-04-01 21:51:12 +00:00
|
|
|
async fn to_stream(
|
|
|
|
&self,
|
2023-09-02 23:30:45 +00:00
|
|
|
identifier: &Arc<str>,
|
2022-04-01 21:51:12 +00:00
|
|
|
from_start: Option<u64>,
|
|
|
|
len: Option<u64>,
|
2023-09-02 23:30:45 +00:00
|
|
|
) -> Result<LocalBoxStream<'static, std::io::Result<Bytes>>, StoreError> {
|
2022-04-01 21:51:12 +00:00
|
|
|
T::to_stream(self, identifier, from_start, len).await
|
|
|
|
}
|
|
|
|
|
|
|
|
async fn read_into<Writer>(
|
|
|
|
&self,
|
2023-09-02 23:30:45 +00:00
|
|
|
identifier: &Arc<str>,
|
2022-04-01 21:51:12 +00:00
|
|
|
writer: &mut Writer,
|
|
|
|
) -> Result<(), std::io::Error>
|
|
|
|
where
|
2022-09-24 22:18:53 +00:00
|
|
|
Writer: AsyncWrite + Unpin,
|
2022-04-01 21:51:12 +00:00
|
|
|
{
|
|
|
|
T::read_into(self, identifier, writer).await
|
|
|
|
}
|
|
|
|
|
2023-09-02 23:30:45 +00:00
|
|
|
async fn len(&self, identifier: &Arc<str>) -> Result<u64, StoreError> {
|
2022-04-01 21:51:12 +00:00
|
|
|
T::len(self, identifier).await
|
|
|
|
}
|
|
|
|
|
2023-09-02 23:30:45 +00:00
|
|
|
async fn remove(&self, identifier: &Arc<str>) -> Result<(), StoreError> {
|
2022-04-01 21:51:12 +00:00
|
|
|
T::remove(self, identifier).await
|
|
|
|
}
|
|
|
|
}
|
2023-08-13 19:12:38 +00:00
|
|
|
|
2023-09-02 23:30:45 +00:00
|
|
|
impl<'a, T> Store for &'a T
|
|
|
|
where
|
|
|
|
T: Store,
|
|
|
|
{
|
|
|
|
async fn health_check(&self) -> Result<(), StoreError> {
|
|
|
|
T::health_check(self).await
|
|
|
|
}
|
|
|
|
|
|
|
|
async fn save_async_read<Reader>(
|
|
|
|
&self,
|
|
|
|
reader: Reader,
|
|
|
|
content_type: mime::Mime,
|
|
|
|
) -> Result<Arc<str>, StoreError>
|
2023-08-13 19:12:38 +00:00
|
|
|
where
|
2023-09-02 23:30:45 +00:00
|
|
|
Reader: AsyncRead + Unpin + 'static,
|
2023-08-13 19:12:38 +00:00
|
|
|
{
|
2023-09-02 23:30:45 +00:00
|
|
|
T::save_async_read(self, reader, content_type).await
|
2023-08-13 19:12:38 +00:00
|
|
|
}
|
|
|
|
|
2023-09-02 23:30:45 +00:00
|
|
|
async fn save_stream<S>(
|
|
|
|
&self,
|
|
|
|
stream: S,
|
|
|
|
content_type: mime::Mime,
|
|
|
|
) -> Result<Arc<str>, StoreError>
|
2023-08-16 00:19:03 +00:00
|
|
|
where
|
2023-09-02 23:30:45 +00:00
|
|
|
S: Stream<Item = std::io::Result<Bytes>> + Unpin + 'static,
|
2023-08-16 00:19:03 +00:00
|
|
|
{
|
2023-09-02 23:30:45 +00:00
|
|
|
T::save_stream(self, stream, content_type).await
|
2023-08-16 00:19:03 +00:00
|
|
|
}
|
|
|
|
|
2023-09-02 23:30:45 +00:00
|
|
|
async fn save_bytes(
|
|
|
|
&self,
|
|
|
|
bytes: Bytes,
|
|
|
|
content_type: mime::Mime,
|
|
|
|
) -> Result<Arc<str>, StoreError> {
|
|
|
|
T::save_bytes(self, bytes, content_type).await
|
2023-08-13 19:12:38 +00:00
|
|
|
}
|
|
|
|
|
2023-09-02 23:30:45 +00:00
|
|
|
fn public_url(&self, identifier: &Arc<str>) -> Option<url::Url> {
|
|
|
|
T::public_url(self, identifier)
|
2023-08-13 19:12:38 +00:00
|
|
|
}
|
2023-08-16 00:19:03 +00:00
|
|
|
|
2023-09-02 23:30:45 +00:00
|
|
|
async fn to_stream(
|
|
|
|
&self,
|
|
|
|
identifier: &Arc<str>,
|
|
|
|
from_start: Option<u64>,
|
|
|
|
len: Option<u64>,
|
|
|
|
) -> Result<LocalBoxStream<'static, std::io::Result<Bytes>>, StoreError> {
|
|
|
|
T::to_stream(self, identifier, from_start, len).await
|
2023-08-16 00:19:03 +00:00
|
|
|
}
|
|
|
|
|
2023-09-02 23:30:45 +00:00
|
|
|
async fn read_into<Writer>(
|
|
|
|
&self,
|
|
|
|
identifier: &Arc<str>,
|
|
|
|
writer: &mut Writer,
|
|
|
|
) -> Result<(), std::io::Error>
|
2023-08-16 00:19:03 +00:00
|
|
|
where
|
2023-09-02 23:30:45 +00:00
|
|
|
Writer: AsyncWrite + Unpin,
|
2023-08-16 00:19:03 +00:00
|
|
|
{
|
2023-09-02 23:30:45 +00:00
|
|
|
T::read_into(self, identifier, writer).await
|
2023-08-16 00:19:03 +00:00
|
|
|
}
|
|
|
|
|
2023-09-02 23:30:45 +00:00
|
|
|
async fn len(&self, identifier: &Arc<str>) -> Result<u64, StoreError> {
|
|
|
|
T::len(self, identifier).await
|
2023-08-16 00:19:03 +00:00
|
|
|
}
|
|
|
|
|
2023-09-02 23:30:45 +00:00
|
|
|
async fn remove(&self, identifier: &Arc<str>) -> Result<(), StoreError> {
|
|
|
|
T::remove(self, identifier).await
|
2023-08-16 00:19:03 +00:00
|
|
|
}
|
|
|
|
}
|