mirror of
https://git.asonix.dog/asonix/pict-rs
synced 2024-12-22 19:31:35 +00:00
Consolidate endpoints for downloading media, cached or backgrounded
This commit is contained in:
parent
07c61cbe87
commit
55d5e43cd2
5 changed files with 81 additions and 72 deletions
13
README.md
13
README.md
|
@ -243,8 +243,6 @@ pict-rs offers the following endpoints:
|
|||
"msg": "ok"
|
||||
}
|
||||
```
|
||||
- `GET /image/download?url=...` Download an image from a remote server, returning the same JSON
|
||||
payload as the `POST` endpoint
|
||||
- `POST /image/backgrounded` Upload an image, like the `/image` endpoint, but don't wait to validate and process it.
|
||||
This endpoint returns the following JSON structure on success with a 202 Accepted status
|
||||
```json
|
||||
|
@ -260,7 +258,16 @@ pict-rs offers the following endpoints:
|
|||
"msg": "ok"
|
||||
}
|
||||
```
|
||||
- `GET /image/backgrounded/claim?upload_id=` Wait for a backgrounded upload to complete, claiming it's result
|
||||
- `GET /image/download?url={url}&backgrounded=(true|false)&ephemeral=(true|false)` Download an image
|
||||
from a remote server, returning the same JSON payload as the `POST` endpoint by default.
|
||||
|
||||
if `backgrounded` is set to `true`, then the ingest processing will be queued for later and the
|
||||
response json will be the same as the `/image/backgrounded` endpoint.
|
||||
|
||||
if `ephemeral` is set to true, the downloaded image will be marked as a "cached" image, and
|
||||
automatically removed from pict-rs N hours after its last access. The duration is configurable
|
||||
with the `--media-cache-duration` run flag, or the `[media] cache_duration` toml option.
|
||||
- `GET /image/backgrounded/claim?upload_id={uuid}` Wait for a backgrounded upload to complete, claiming it's result
|
||||
Possible results:
|
||||
- 200 Ok (validation and ingest complete):
|
||||
```json
|
||||
|
|
|
@ -57,6 +57,7 @@ pub(crate) async fn ingest<R, S>(
|
|||
stream: impl Stream<Item = Result<Bytes, Error>>,
|
||||
declared_alias: Option<Alias>,
|
||||
should_validate: bool,
|
||||
is_cached: bool,
|
||||
) -> Result<Session<R, S>, Error>
|
||||
where
|
||||
R: FullRepo + 'static,
|
||||
|
@ -95,9 +96,9 @@ where
|
|||
save_upload(repo, store, &hash, &identifier).await?;
|
||||
|
||||
if let Some(alias) = declared_alias {
|
||||
session.add_existing_alias(&hash, alias).await?
|
||||
session.add_existing_alias(&hash, alias, is_cached).await?
|
||||
} else {
|
||||
session.create_alias(&hash, input_type).await?;
|
||||
session.create_alias(&hash, input_type, is_cached).await?;
|
||||
}
|
||||
|
||||
Ok(session)
|
||||
|
@ -161,7 +162,12 @@ where
|
|||
}
|
||||
|
||||
#[tracing::instrument]
|
||||
async fn add_existing_alias(&mut self, hash: &[u8], alias: Alias) -> Result<(), Error> {
|
||||
async fn add_existing_alias(
|
||||
&mut self,
|
||||
hash: &[u8],
|
||||
alias: Alias,
|
||||
is_cached: bool,
|
||||
) -> Result<(), Error> {
|
||||
AliasRepo::create(&self.repo, &alias)
|
||||
.await?
|
||||
.map_err(|_| UploadError::DuplicateAlias)?;
|
||||
|
@ -171,11 +177,20 @@ where
|
|||
self.repo.relate_hash(&alias, hash.to_vec().into()).await?;
|
||||
self.repo.relate_alias(hash.to_vec().into(), &alias).await?;
|
||||
|
||||
if is_cached {
|
||||
self.repo.mark_cached(&alias).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument]
|
||||
async fn create_alias(&mut self, hash: &[u8], input_type: ValidInputType) -> Result<(), Error> {
|
||||
async fn create_alias(
|
||||
&mut self,
|
||||
hash: &[u8],
|
||||
input_type: ValidInputType,
|
||||
is_cached: bool,
|
||||
) -> Result<(), Error> {
|
||||
tracing::debug!("Alias gen loop");
|
||||
|
||||
loop {
|
||||
|
@ -187,6 +202,10 @@ where
|
|||
self.repo.relate_hash(&alias, hash.to_vec().into()).await?;
|
||||
self.repo.relate_alias(hash.to_vec().into(), &alias).await?;
|
||||
|
||||
if is_cached {
|
||||
self.repo.mark_cached(&alias).await?;
|
||||
}
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
|
|
96
src/main.rs
96
src/main.rs
|
@ -164,7 +164,7 @@ async fn upload_backgrounded<R: FullRepo, S: Store>(
|
|||
.expect("Identifier exists")
|
||||
.to_bytes()?;
|
||||
|
||||
queue::queue_ingest(&**repo, identifier, upload_id, None, true).await?;
|
||||
queue::queue_ingest(&**repo, identifier, upload_id, None, true, false).await?;
|
||||
|
||||
files.push(serde_json::json!({
|
||||
"upload_id": upload_id.to_string(),
|
||||
|
@ -223,6 +223,12 @@ async fn claim_upload<R: FullRepo>(
|
|||
#[derive(Debug, serde::Deserialize)]
|
||||
struct UrlQuery {
|
||||
url: String,
|
||||
|
||||
#[serde(default)]
|
||||
backgrounded: bool,
|
||||
|
||||
#[serde(default)]
|
||||
ephemeral: bool,
|
||||
}
|
||||
|
||||
/// download an image from a URL
|
||||
|
@ -233,7 +239,22 @@ async fn download<R: FullRepo + 'static, S: Store + 'static>(
|
|||
store: web::Data<S>,
|
||||
query: web::Query<UrlQuery>,
|
||||
) -> Result<HttpResponse, Error> {
|
||||
let res = client.get(&query.url).send().await?;
|
||||
if query.backgrounded {
|
||||
do_download_backgrounded(client, repo, store, &query.url, query.ephemeral).await
|
||||
} else {
|
||||
do_download_inline(client, repo, store, &query.url, query.ephemeral).await
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(name = "Downloading file inline", skip(client, repo))]
|
||||
async fn do_download_inline<R: FullRepo + 'static, S: Store + 'static>(
|
||||
client: web::Data<Client>,
|
||||
repo: web::Data<R>,
|
||||
store: web::Data<S>,
|
||||
url: &str,
|
||||
is_cached: bool,
|
||||
) -> Result<HttpResponse, Error> {
|
||||
let res = client.get(url).send().await?;
|
||||
|
||||
if !res.status().is_success() {
|
||||
return Err(UploadError::Download(res.status()).into());
|
||||
|
@ -243,7 +264,7 @@ async fn download<R: FullRepo + 'static, S: Store + 'static>(
|
|||
.map_err(Error::from)
|
||||
.limit((CONFIG.media.max_file_size * MEGABYTES) as u64);
|
||||
|
||||
let mut session = ingest::ingest(&**repo, &**store, stream, None, true).await?;
|
||||
let mut session = ingest::ingest(&**repo, &**store, stream, None, true, is_cached).await?;
|
||||
|
||||
let alias = session.alias().expect("alias should exist").to_owned();
|
||||
let delete_token = session.delete_token().await?;
|
||||
|
@ -262,6 +283,7 @@ async fn download<R: FullRepo + 'static, S: Store + 'static>(
|
|||
};
|
||||
|
||||
session.disarm();
|
||||
|
||||
Ok(HttpResponse::Created().json(&serde_json::json!({
|
||||
"msg": "ok",
|
||||
"files": [{
|
||||
|
@ -272,14 +294,15 @@ async fn download<R: FullRepo + 'static, S: Store + 'static>(
|
|||
})))
|
||||
}
|
||||
|
||||
#[instrument(name = "Downloading file for background", skip(client))]
|
||||
async fn download_backgrounded<R: FullRepo + 'static, S: Store + 'static>(
|
||||
#[instrument(name = "Downloading file in background", skip(client))]
|
||||
async fn do_download_backgrounded<R: FullRepo + 'static, S: Store + 'static>(
|
||||
client: web::Data<Client>,
|
||||
repo: web::Data<R>,
|
||||
store: web::Data<S>,
|
||||
query: web::Query<UrlQuery>,
|
||||
url: &str,
|
||||
is_cached: bool,
|
||||
) -> Result<HttpResponse, Error> {
|
||||
let res = client.get(&query.url).send().await?;
|
||||
let res = client.get(url).send().await?;
|
||||
|
||||
if !res.status().is_success() {
|
||||
return Err(UploadError::Download(res.status()).into());
|
||||
|
@ -297,7 +320,7 @@ async fn download_backgrounded<R: FullRepo + 'static, S: Store + 'static>(
|
|||
.expect("Identifier exists")
|
||||
.to_bytes()?;
|
||||
|
||||
queue::queue_ingest(&**repo, identifier, upload_id, None, true).await?;
|
||||
queue::queue_ingest(&**repo, identifier, upload_id, None, true, is_cached).await?;
|
||||
|
||||
Ok(HttpResponse::Accepted().json(&serde_json::json!({
|
||||
"msg": "ok",
|
||||
|
@ -307,55 +330,6 @@ async fn download_backgrounded<R: FullRepo + 'static, S: Store + 'static>(
|
|||
})))
|
||||
}
|
||||
|
||||
/// cache an image from a URL
|
||||
#[instrument(name = "Caching file", skip(client, repo))]
|
||||
async fn cache<R: FullRepo + 'static, S: Store + 'static>(
|
||||
client: web::Data<Client>,
|
||||
repo: web::Data<R>,
|
||||
store: web::Data<S>,
|
||||
query: web::Query<UrlQuery>,
|
||||
) -> Result<HttpResponse, Error> {
|
||||
let res = client.get(&query.url).send().await?;
|
||||
|
||||
if !res.status().is_success() {
|
||||
return Err(UploadError::Download(res.status()).into());
|
||||
}
|
||||
|
||||
let stream = res
|
||||
.map_err(Error::from)
|
||||
.limit((CONFIG.media.max_file_size * MEGABYTES) as u64);
|
||||
|
||||
let mut session = ingest::ingest(&**repo, &**store, stream, None, true).await?;
|
||||
|
||||
let alias = session.alias().expect("alias should exist").to_owned();
|
||||
let delete_token = session.delete_token().await?;
|
||||
|
||||
let identifier = repo.identifier_from_alias::<S::Identifier>(&alias).await?;
|
||||
|
||||
let details = repo.details(&identifier).await?;
|
||||
|
||||
let details = if let Some(details) = details {
|
||||
details
|
||||
} else {
|
||||
let hint = details_hint(&alias);
|
||||
let new_details = Details::from_store((**store).clone(), identifier.clone(), hint).await?;
|
||||
repo.relate_details(&identifier, &new_details).await?;
|
||||
new_details
|
||||
};
|
||||
|
||||
repo.mark_cached(&alias).await?;
|
||||
|
||||
session.disarm();
|
||||
Ok(HttpResponse::Created().json(&serde_json::json!({
|
||||
"msg": "ok",
|
||||
"files": [{
|
||||
"file": alias.to_string(),
|
||||
"delete_token": delete_token.to_string(),
|
||||
"details": details,
|
||||
}]
|
||||
})))
|
||||
}
|
||||
|
||||
/// Delete aliases and files
|
||||
#[instrument(name = "Deleting file", skip(repo))]
|
||||
async fn delete<R: FullRepo>(
|
||||
|
@ -757,7 +731,7 @@ async fn launch<R: FullRepo + Clone + 'static, S: Store + Clone + 'static>(
|
|||
let stream = stream.map_err(Error::from);
|
||||
|
||||
Box::pin(
|
||||
async move { ingest::ingest(&repo, &store, stream, None, true).await }
|
||||
async move { ingest::ingest(&repo, &store, stream, None, true, false).await }
|
||||
.instrument(span),
|
||||
)
|
||||
})),
|
||||
|
@ -790,6 +764,7 @@ async fn launch<R: FullRepo + Clone + 'static, S: Store + Clone + 'static>(
|
|||
stream,
|
||||
Some(Alias::from_existing(&filename)),
|
||||
!CONFIG.media.skip_validate_imports,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
@ -869,11 +844,6 @@ async fn launch<R: FullRepo + Clone + 'static, S: Store + Clone + 'static>(
|
|||
),
|
||||
)
|
||||
.service(web::resource("/download").route(web::get().to(download::<R, S>)))
|
||||
.service(
|
||||
web::resource("/download_backgrounded")
|
||||
.route(web::get().to(download_backgrounded::<R, S>)),
|
||||
)
|
||||
.service(web::resource("/cache").route(web::get().to(cache::<R, S>)))
|
||||
.service(
|
||||
web::resource("/delete/{delete_token}/{filename}")
|
||||
.route(web::delete().to(delete::<R>))
|
||||
|
|
|
@ -37,6 +37,7 @@ enum Process {
|
|||
upload_id: Serde<UploadId>,
|
||||
declared_alias: Option<Serde<Alias>>,
|
||||
should_validate: bool,
|
||||
is_cached: bool,
|
||||
},
|
||||
Generate {
|
||||
target_format: ImageFormat,
|
||||
|
@ -84,12 +85,14 @@ pub(crate) async fn queue_ingest<R: QueueRepo>(
|
|||
upload_id: UploadId,
|
||||
declared_alias: Option<Alias>,
|
||||
should_validate: bool,
|
||||
is_cached: bool,
|
||||
) -> Result<(), Error> {
|
||||
let job = serde_json::to_vec(&Process::Ingest {
|
||||
identifier,
|
||||
declared_alias: declared_alias.map(Serde::new),
|
||||
upload_id: Serde::new(upload_id),
|
||||
should_validate,
|
||||
is_cached,
|
||||
})?;
|
||||
repo.push(PROCESS_QUEUE, job.into()).await?;
|
||||
Ok(())
|
||||
|
|
|
@ -27,6 +27,7 @@ where
|
|||
upload_id,
|
||||
declared_alias,
|
||||
should_validate,
|
||||
is_cached,
|
||||
} => {
|
||||
process_ingest(
|
||||
repo,
|
||||
|
@ -35,6 +36,7 @@ where
|
|||
Serde::into_inner(upload_id),
|
||||
declared_alias.map(Serde::into_inner),
|
||||
should_validate,
|
||||
is_cached,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
|
@ -72,6 +74,7 @@ async fn process_ingest<R, S>(
|
|||
upload_id: UploadId,
|
||||
declared_alias: Option<Alias>,
|
||||
should_validate: bool,
|
||||
is_cached: bool,
|
||||
) -> Result<(), Error>
|
||||
where
|
||||
R: FullRepo + 'static,
|
||||
|
@ -85,8 +88,15 @@ where
|
|||
.await?
|
||||
.map_err(Error::from);
|
||||
|
||||
let session =
|
||||
crate::ingest::ingest(repo, store, stream, declared_alias, should_validate).await?;
|
||||
let session = crate::ingest::ingest(
|
||||
repo,
|
||||
store,
|
||||
stream,
|
||||
declared_alias,
|
||||
should_validate,
|
||||
is_cached,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let token = session.delete_token().await?;
|
||||
|
||||
|
|
Loading…
Reference in a new issue