LemmyNet/pict-rs
2
0
Fork 0
mirror of https://git.asonix.dog/asonix/pict-rs synced 2024-12-22 03:11:24 +00:00
Code Releases Activity

Include file extension in identifiers

Browse source
This commit is contained in:
asonix 2024-02-27 20:41:25 -06:00
parent d13f7fe969
commit 04dcc9a0c8
10 changed files with 96 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/backgrounded.rs
View file

@ -59,7 +59,9 @@ impl Backgrounded {
});
// use octet-stream, we don't know the upload's real type yet
let identifier = store.save_stream(stream, APPLICATION_OCTET_STREAM).await?;
let identifier = store
.save_stream(stream, APPLICATION_OCTET_STREAM, None)
.await?;
self.identifier = Some(identifier);

4
src/details.rs
View file

@ -116,6 +116,10 @@ impl Details {
(*self.inner.content_type).clone()
}
pub(crate) fn file_extension(&self) -> &'static str {
self.inner.format.file_extension()
}
pub(crate) fn system_time(&self) -> std::time::SystemTime {
self.inner.created_at.into()
}

2
src/formats/image.rs
View file

@ -70,7 +70,7 @@ impl ImageFormat {
}
}
pub(super) const fn file_extension(self) -> &'static str {
pub(crate) const fn file_extension(self) -> &'static str {
match self {
Self::Avif => ".avif",
Self::Jpeg => ".jpeg",

22
src/generate.rs
View file

@ -133,7 +133,11 @@ async fn process<S: Store + 'static>(
let identifier = state
.store
.save_stream(bytes.into_io_stream(), details.media_type())
.save_stream(
bytes.into_io_stream(),
details.media_type(),
Some(details.file_extension()),
)
.await?;
if let Err(VariantAlreadyExists) = state
@ -173,7 +177,7 @@ where
.await?
.ok_or(UploadError::MissingIdentifier)?;
let (reader, media_type) =
let (reader, media_type, file_extension) =
if let Some(processable_format) = original_details.internal_format().processable_format() {
let thumbnail_format = state.config.media.image.format.unwrap_or(ImageFormat::Webp);
@ -185,6 +189,7 @@ where
(
process.drive_with_stream(stream),
thumbnail_format.media_type(),
thumbnail_format.file_extension(),
)
} else {
let thumbnail_format = match state.config.media.image.format {
@ -205,11 +210,20 @@ where
)
.await?;
(reader, thumbnail_format.media_type())
(
reader,
thumbnail_format.media_type(),
thumbnail_format.file_extension(),
)
};
let motion_identifier = reader
.with_stdout(|stdout| async { state.store.save_async_read(stdout, media_type).await })
.with_stdout(|stdout| async {
state
.store
.save_async_read(stdout, media_type, Some(file_extension))
.await
})
.await??;
state

12
src/generate/ffmpeg.rs
View file

@ -18,7 +18,7 @@ pub(super) enum ThumbnailFormat {
}
impl ThumbnailFormat {
const fn as_ffmpeg_codec(self) -> &'static str {
const fn ffmpeg_codec(self) -> &'static str {
match self {
Self::Jpeg => "mjpeg",
Self::Png => "png",
@ -26,7 +26,7 @@ impl ThumbnailFormat {
}
}
const fn to_file_extension(self) -> &'static str {
pub(super) const fn file_extension(self) -> &'static str {
match self {
Self::Jpeg => ".jpeg",
Self::Png => ".png",
@ -34,7 +34,7 @@ impl ThumbnailFormat {
}
}
const fn as_ffmpeg_format(self) -> &'static str {
const fn ffmpeg_format(self) -> &'static str {
match self {
Self::Jpeg | Self::Png => "image2",
Self::Webp => "webp",
@ -57,7 +57,7 @@ pub(super) async fn thumbnail<S: Store>(
input_format: InternalVideoFormat,
format: ThumbnailFormat,
) -> Result<ProcessRead, FfMpegError> {
let output_file = state.tmp_dir.tmp_file(Some(format.to_file_extension()));
let output_file = state.tmp_dir.tmp_file(Some(format.file_extension()));
crate::store::file_store::safe_create_parent(&output_file)
.await
@ -90,9 +90,9 @@ pub(super) async fn thumbnail<S: Store>(
"-frames:v".as_ref(),
"1".as_ref(),
"-codec".as_ref(),
format.as_ffmpeg_codec().as_ref(),
format.ffmpeg_codec().as_ref(),
"-f".as_ref(),
format.as_ffmpeg_format().as_ref(),
format.ffmpeg_format().as_ref(),
output_path,
],
&[],

12
src/ingest.rs
View file

@ -88,7 +88,11 @@ where
state
.store
.save_async_read(hasher_reader, input_type.media_type())
.save_async_read(
hasher_reader,
input_type.media_type(),
Some(input_type.file_extension()),
)
.await
.map(move |identifier| (hash_state, identifier))
})
@ -131,7 +135,11 @@ where
let identifier = state
.store
.save_async_read(hasher_reader, input_type.media_type())
.save_async_read(
hasher_reader,
input_type.media_type(),
Some(input_type.file_extension()),
)
.await?;
let details = Details::danger_dummy(input_type);

2
src/migrate_store.rs
View file

@ -409,7 +409,7 @@ where
let new_identifier = to
.store
.save_stream(stream, details.media_type())
.save_stream(stream, details.media_type(), Some(details.file_extension()))
.await
.map_err(MigrateError::To)?;

20
src/store.rs
View file

@ -89,6 +89,7 @@ pub(crate) trait Store: Clone + Debug {
&self,
reader: Reader,
content_type: mime::Mime,
extension: Option<&str>,
) -> Result<Arc<str>, StoreError>
where
Reader: AsyncRead;
@ -97,6 +98,7 @@ pub(crate) trait Store: Clone + Debug {
&self,
stream: S,
content_type: mime::Mime,
extension: Option<&str>,
) -> Result<Arc<str>, StoreError>
where
S: Stream<Item = std::io::Result<Bytes>>;
@ -148,22 +150,24 @@ where
&self,
reader: Reader,
content_type: mime::Mime,
extension: Option<&str>,
) -> Result<Arc<str>, StoreError>
where
Reader: AsyncRead,
{
T::save_async_read(self, reader, content_type).await
T::save_async_read(self, reader, content_type, extension).await
}
async fn save_stream<S>(
&self,
stream: S,
content_type: mime::Mime,
extension: Option<&str>,
) -> Result<Arc<str>, StoreError>
where
S: Stream<Item = std::io::Result<Bytes>>,
{
T::save_stream(self, stream, content_type).await
T::save_stream(self, stream, content_type, extension).await
}
fn public_url(&self, identifier: &Arc<str>) -> Option<url::Url> {
@ -211,22 +215,24 @@ where
&self,
reader: Reader,
content_type: mime::Mime,
extension: Option<&str>,
) -> Result<Arc<str>, StoreError>
where
Reader: AsyncRead,
{
T::save_async_read(self, reader, content_type).await
T::save_async_read(self, reader, content_type, extension).await
}
async fn save_stream<S>(
&self,
stream: S,
content_type: mime::Mime,
extension: Option<&str>,
) -> Result<Arc<str>, StoreError>
where
S: Stream<Item = std::io::Result<Bytes>>,
{
T::save_stream(self, stream, content_type).await
T::save_stream(self, stream, content_type, extension).await
}
fn public_url(&self, identifier: &Arc<str>) -> Option<url::Url> {
@ -274,22 +280,24 @@ where
&self,
reader: Reader,
content_type: mime::Mime,
extension: Option<&str>,
) -> Result<Arc<str>, StoreError>
where
Reader: AsyncRead,
{
T::save_async_read(self, reader, content_type).await
T::save_async_read(self, reader, content_type, extension).await
}
async fn save_stream<S>(
&self,
stream: S,
content_type: mime::Mime,
extension: Option<&str>,
) -> Result<Arc<str>, StoreError>
where
S: Stream<Item = std::io::Result<Bytes>>,
{
T::save_stream(self, stream, content_type).await
T::save_stream(self, stream, content_type, extension).await
}
fn public_url(&self, identifier: &Arc<str>) -> Option<url::Url> {

15
src/store/file_store.rs
View file

@ -56,13 +56,14 @@ impl Store for FileStore {
&self,
reader: Reader,
_content_type: mime::Mime,
extension: Option<&str>,
) -> Result<Arc<str>, StoreError>
where
Reader: AsyncRead,
{
let mut reader = std::pin::pin!(reader);
let path = self.next_file();
let path = self.next_file(extension);
if let Err(e) = self.safe_save_reader(&path, &mut reader).await {
self.safe_remove_file(&path).await?;
@ -76,11 +77,12 @@ impl Store for FileStore {
&self,
stream: S,
content_type: mime::Mime,
extension: Option<&str>,
) -> Result<Arc<str>, StoreError>
where
S: Stream<Item = std::io::Result<Bytes>>,
{
self.save_async_read(StreamReader::new(stream), content_type)
self.save_async_read(StreamReader::new(stream), content_type, extension)
.await
}
@ -169,9 +171,14 @@ impl FileStore {
self.root_dir.join(file_id.as_ref())
}
fn next_file(&self) -> PathBuf {
fn next_file(&self, extension: Option<&str>) -> PathBuf {
let target_path = crate::file_path::generate_disk(self.root_dir.clone());
let filename = uuid::Uuid::new_v4().to_string();
let file_id = uuid::Uuid::new_v4().to_string();
let filename = if let Some(ext) = extension {
file_id + ext
} else {
file_id
};
target_path.join(filename)
}

37
src/store/object_store.rs
View file

@ -211,12 +211,17 @@ impl Store for ObjectStore {
&self,
reader: Reader,
content_type: mime::Mime,
extension: Option<&str>,
) -> Result<Arc<str>, StoreError>
where
Reader: AsyncRead,
{
self.save_stream(ReaderStream::with_capacity(reader, 1024 * 64), content_type)
.await
self.save_stream(
ReaderStream::with_capacity(reader, 1024 * 64),
content_type,
extension,
)
.await
}
#[tracing::instrument(skip_all)]
@ -224,14 +229,18 @@ impl Store for ObjectStore {
&self,
stream: S,
content_type: mime::Mime,
extension: Option<&str>,
) -> Result<Arc<str>, StoreError>
where
S: Stream<Item = std::io::Result<Bytes>>,
{
match self.start_upload(stream, content_type.clone()).await? {
match self
.start_upload(stream, content_type.clone(), extension)
.await?
{
UploadState::Single(first_chunk) => {
let (req, object_id) = self
.put_object_request(first_chunk.len(), content_type)
.put_object_request(first_chunk.len(), content_type, extension)
.await?;
let response = req
@ -447,6 +456,7 @@ impl ObjectStore {
&self,
stream: S,
content_type: mime::Mime,
extension: Option<&str>,
) -> Result<UploadState, StoreError>
where
S: Stream<Item = std::io::Result<Bytes>>,
@ -461,7 +471,9 @@ impl ObjectStore {
let mut first_chunk = Some(first_chunk);
let (req, object_id) = self.create_multipart_request(content_type).await?;
let (req, object_id) = self
.create_multipart_request(content_type, extension)
.await?;
let response = req
.send()
.with_metrics(crate::init_metrics::OBJECT_STORAGE_CREATE_MULTIPART_REQUEST)
@ -574,8 +586,9 @@ impl ObjectStore {
&self,
length: usize,
content_type: mime::Mime,
extension: Option<&str>,
) -> Result<(RequestBuilder, Arc<str>), StoreError> {
let path = self.next_file();
let path = self.next_file(extension);
let mut action = self.bucket.put_object(Some(&self.credentials), &path);
@ -592,8 +605,9 @@ impl ObjectStore {
async fn create_multipart_request(
&self,
content_type: mime::Mime,
extension: Option<&str>,
) -> Result<(RequestBuilder, Arc<str>), StoreError> {
let path = self.next_file();
let path = self.next_file(extension);
let mut action = self
.bucket
@ -763,9 +777,14 @@ impl ObjectStore {
self.build_request(action)
}
fn next_file(&self) -> String {
fn next_file(&self, extension: Option<&str>) -> String {
let path = crate::file_path::generate_object();
let filename = uuid::Uuid::new_v4().to_string();
let file_id = uuid::Uuid::new_v4().to_string();
let filename = if let Some(ext) = extension {
file_id + ext
} else {
file_id
};
format!("{path}/{filename}")
}