Don't fetch metadata in background for local API requests.

This commit is contained in:
Dessalines 2024-05-06 08:34:40 -04:00
parent a38830631d
commit dde7a44e57
4 changed files with 93 additions and 71 deletions

View file

@ -66,14 +66,89 @@ pub async fn fetch_link_metadata(url: &Url, context: &LemmyContext) -> LemmyResu
}) })
} }
/// Generate post thumbnail in background task, because some sites can be very slow to respond. /// Generates and saves a post thumbnail and metadata.
/// ///
/// Takes a callback to generate a send activity task, so that post can be federated with metadata. /// Takes a callback to generate a send activity task, so that post can be federated with metadata.
/// ///
/// TODO: `federated_thumbnail` param can be removed once we federate full metadata and can /// TODO: `federated_thumbnail` param can be removed once we federate full metadata and can
/// write it to db directly, without calling this function. /// write it to db directly, without calling this function.
/// https://github.com/LemmyNet/lemmy/issues/4598 /// https://github.com/LemmyNet/lemmy/issues/4598
pub fn generate_post_link_metadata( pub async fn generate_post_link_metadata(
post: Post,
custom_thumbnail: Option<Url>,
federated_thumbnail: Option<Url>,
send_activity: impl FnOnce(Post) -> Option<SendActivityData> + Send + 'static,
local_site: Option<LocalSite>,
context: Data<LemmyContext>,
) -> LemmyResult<()> {
let metadata = match &post.url {
Some(url) => fetch_link_metadata(url, &context).await.unwrap_or_default(),
_ => Default::default(),
};
let is_image_post = metadata
.content_type
.as_ref()
.is_some_and(|content_type| content_type.starts_with("image"));
// Decide if we are allowed to generate local thumbnail
let allow_sensitive = local_site_opt_to_sensitive(&local_site);
let allow_generate_thumbnail = allow_sensitive || !post.nsfw;
let thumbnail_url = if is_image_post {
if allow_generate_thumbnail {
match post.url {
Some(url) => generate_pictrs_thumbnail(&url, &context)
.await
.ok()
.map(Into::into),
None => None,
}
} else {
None
}
} else {
// Use custom thumbnail if available and its not an image post
if let Some(custom_thumbnail) = custom_thumbnail {
proxy_image_link(custom_thumbnail, &context).await.ok()
}
// Use federated thumbnail if available
else if let Some(federated_thumbnail) = federated_thumbnail {
proxy_image_link(federated_thumbnail, &context).await.ok()
}
// Generate local thumbnail if allowed
else if allow_generate_thumbnail {
match metadata.opengraph_data.image {
Some(url) => generate_pictrs_thumbnail(&url, &context)
.await
.ok()
.map(Into::into),
None => None,
}
}
// Otherwise use opengraph preview image directly
else {
metadata.opengraph_data.image
}
};
let form = PostUpdateForm {
embed_title: Some(metadata.opengraph_data.title),
embed_description: Some(metadata.opengraph_data.description),
embed_video_url: Some(metadata.opengraph_data.embed_video_url),
thumbnail_url: Some(thumbnail_url),
url_content_type: Some(metadata.content_type),
..Default::default()
};
let updated_post = Post::update(&mut context.pool(), post.id, &form).await?;
if let Some(send_activity) = send_activity(updated_post) {
ActivityChannel::submit_activity(send_activity, &context).await?;
}
Ok(())
}
/// Generates a post thumbnail in background task, because some sites can be very slow to respond.
pub fn generate_post_link_metadata_background(
post: Post, post: Post,
custom_thumbnail: Option<Url>, custom_thumbnail: Option<Url>,
federated_thumbnail: Option<Url>, federated_thumbnail: Option<Url>,
@ -82,71 +157,16 @@ pub fn generate_post_link_metadata(
context: Data<LemmyContext>, context: Data<LemmyContext>,
) { ) {
spawn_try_task(async move { spawn_try_task(async move {
let metadata = match &post.url { generate_post_link_metadata(
Some(url) => fetch_link_metadata(url, &context).await.unwrap_or_default(), post,
_ => Default::default(), custom_thumbnail,
}; federated_thumbnail,
send_activity,
let is_image_post = metadata local_site,
.content_type context,
.as_ref() )
.is_some_and(|content_type| content_type.starts_with("image")); .await
})
// Decide if we are allowed to generate local thumbnail
let allow_sensitive = local_site_opt_to_sensitive(&local_site);
let allow_generate_thumbnail = allow_sensitive || !post.nsfw;
let thumbnail_url = if is_image_post {
if allow_generate_thumbnail {
match post.url {
Some(url) => generate_pictrs_thumbnail(&url, &context)
.await
.ok()
.map(Into::into),
None => None,
}
} else {
None
}
} else {
// Use custom thumbnail if available and its not an image post
if let Some(custom_thumbnail) = custom_thumbnail {
proxy_image_link(custom_thumbnail, &context).await.ok()
}
// Use federated thumbnail if available
else if let Some(federated_thumbnail) = federated_thumbnail {
proxy_image_link(federated_thumbnail, &context).await.ok()
}
// Generate local thumbnail if allowed
else if allow_generate_thumbnail {
match metadata.opengraph_data.image {
Some(url) => generate_pictrs_thumbnail(&url, &context)
.await
.ok()
.map(Into::into),
None => None,
}
}
// Otherwise use opengraph preview image directly
else {
metadata.opengraph_data.image
}
};
let form = PostUpdateForm {
embed_title: Some(metadata.opengraph_data.title),
embed_description: Some(metadata.opengraph_data.description),
embed_video_url: Some(metadata.opengraph_data.embed_video_url),
thumbnail_url: Some(thumbnail_url),
url_content_type: Some(metadata.content_type),
..Default::default()
};
let updated_post = Post::update(&mut context.pool(), post.id, &form).await?;
if let Some(send_activity) = send_activity(updated_post) {
ActivityChannel::submit_activity(send_activity, &context).await?;
}
Ok(())
});
} }
/// Extract site metadata from HTML Opengraph attributes. /// Extract site metadata from HTML Opengraph attributes.

View file

@ -161,7 +161,8 @@ pub async fn create_post(
|post| Some(SendActivityData::CreatePost(post)), |post| Some(SendActivityData::CreatePost(post)),
Some(local_site), Some(local_site),
context.reset_request_count(), context.reset_request_count(),
); )
.await?;
// They like their own post by default // They like their own post by default
let person_id = local_user_view.person.id; let person_id = local_user_view.person.id;

View file

@ -116,7 +116,8 @@ pub async fn update_post(
|post| Some(SendActivityData::UpdatePost(post)), |post| Some(SendActivityData::UpdatePost(post)),
Some(local_site), Some(local_site),
context.reset_request_count(), context.reset_request_count(),
); )
.await?;
build_post_response( build_post_response(
context.deref(), context.deref(),

View file

@ -24,7 +24,7 @@ use chrono::{DateTime, Utc};
use html2text::{from_read_with_decorator, render::text_renderer::TrivialDecorator}; use html2text::{from_read_with_decorator, render::text_renderer::TrivialDecorator};
use lemmy_api_common::{ use lemmy_api_common::{
context::LemmyContext, context::LemmyContext,
request::generate_post_link_metadata, request::generate_post_link_metadata_background,
utils::{ utils::{
get_url_blocklist, get_url_blocklist,
local_site_opt_to_slur_regex, local_site_opt_to_slur_regex,
@ -278,7 +278,7 @@ impl Object for ApubPost {
let timestamp = page.updated.or(page.published).unwrap_or_else(naive_now); let timestamp = page.updated.or(page.published).unwrap_or_else(naive_now);
let post = Post::insert_apub(&mut context.pool(), timestamp, &form).await?; let post = Post::insert_apub(&mut context.pool(), timestamp, &form).await?;
generate_post_link_metadata( generate_post_link_metadata_background(
post.clone(), post.clone(),
None, None,
page.image.map(|i| i.url), page.image.map(|i| i.url),