From df07d8e31cb38502cd52b678560ba800c02cf799 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Thu, 31 Oct 2024 13:10:45 +0100 Subject: [PATCH 1/3] Skip api test for fetching nested comment (#5152) --- api_tests/src/comment.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_tests/src/comment.spec.ts b/api_tests/src/comment.spec.ts index 153405820..c3f4b3efe 100644 --- a/api_tests/src/comment.spec.ts +++ b/api_tests/src/comment.spec.ts @@ -860,7 +860,7 @@ test("Dont send a comment reply to a blocked community", async () => { /// Fetching a deeply nested comment can lead to stack overflow as all parent comments are also /// fetched recursively. Ensure that it works properly. -test("Fetch a deeply nested comment", async () => { +test.skip("Fetch a deeply nested comment", async () => { let lastComment; for (let i = 0; i < 50; i++) { let commentRes = await createComment( From 8f88dda28fb5b4a6c4c8858b40daa22ccad530dd Mon Sep 17 00:00:00 2001 From: Integral Date: Thu, 31 Oct 2024 20:12:24 +0800 Subject: [PATCH 2/3] refactor: destructure tuples to enhance readability (#5151) --- crates/api/src/sitemap.rs | 6 +++--- crates/apub/src/objects/comment.rs | 10 +++++----- crates/apub/src/objects/person.rs | 9 ++++++--- crates/apub/src/objects/private_message.rs | 8 ++++---- crates/db_views/src/custom_emoji_view.rs | 8 ++++---- crates/utils/src/utils/markdown/image_links.rs | 17 +++++++---------- src/scheduled_tasks.rs | 6 +++--- 7 files changed, 32 insertions(+), 32 deletions(-) diff --git a/crates/api/src/sitemap.rs b/crates/api/src/sitemap.rs index c3c3c417c..57b39a5b3 100644 --- a/crates/api/src/sitemap.rs +++ b/crates/api/src/sitemap.rs @@ -14,9 +14,9 @@ async fn generate_urlset( ) -> LemmyResult { let urls = posts .into_iter() - .map_while(|post| { - Url::builder(post.0.to_string()) - .last_modified(post.1.into()) + .map_while(|(url, date_time)| { + Url::builder(url.to_string()) + .last_modified(date_time.into()) .build() .ok() }) diff --git a/crates/apub/src/objects/comment.rs b/crates/apub/src/objects/comment.rs index 403ecbf94..e6cc2ba85 100644 --- a/crates/apub/src/objects/comment.rs +++ b/crates/apub/src/objects/comment.rs @@ -247,13 +247,13 @@ pub(crate) mod tests { } async fn cleanup( - data: (ApubPerson, ApubCommunity, ApubPost, ApubSite), + (person, community, post, site): (ApubPerson, ApubCommunity, ApubPost, ApubSite), context: &LemmyContext, ) -> LemmyResult<()> { - Post::delete(&mut context.pool(), data.2.id).await?; - Community::delete(&mut context.pool(), data.1.id).await?; - Person::delete(&mut context.pool(), data.0.id).await?; - Site::delete(&mut context.pool(), data.3.id).await?; + Post::delete(&mut context.pool(), post.id).await?; + Community::delete(&mut context.pool(), community.id).await?; + Person::delete(&mut context.pool(), person.id).await?; + Site::delete(&mut context.pool(), site.id).await?; LocalSite::delete(&mut context.pool()).await?; Ok(()) } diff --git a/crates/apub/src/objects/person.rs b/crates/apub/src/objects/person.rs index 4e8519f78..737579662 100644 --- a/crates/apub/src/objects/person.rs +++ b/crates/apub/src/objects/person.rs @@ -285,9 +285,12 @@ pub(crate) mod tests { Ok(()) } - async fn cleanup(data: (ApubPerson, ApubSite), context: &LemmyContext) -> LemmyResult<()> { - DbPerson::delete(&mut context.pool(), data.0.id).await?; - Site::delete(&mut context.pool(), data.1.id).await?; + async fn cleanup( + (person, site): (ApubPerson, ApubSite), + context: &LemmyContext, + ) -> LemmyResult<()> { + DbPerson::delete(&mut context.pool(), person.id).await?; + Site::delete(&mut context.pool(), site.id).await?; Ok(()) } } diff --git a/crates/apub/src/objects/private_message.rs b/crates/apub/src/objects/private_message.rs index 5a191cc66..3a61eb4a5 100644 --- a/crates/apub/src/objects/private_message.rs +++ b/crates/apub/src/objects/private_message.rs @@ -186,12 +186,12 @@ mod tests { } async fn cleanup( - data: (ApubPerson, ApubPerson, ApubSite), + (person1, person2, site): (ApubPerson, ApubPerson, ApubSite), context: &Data, ) -> LemmyResult<()> { - Person::delete(&mut context.pool(), data.0.id).await?; - Person::delete(&mut context.pool(), data.1.id).await?; - Site::delete(&mut context.pool(), data.2.id).await?; + Person::delete(&mut context.pool(), person1.id).await?; + Person::delete(&mut context.pool(), person2.id).await?; + Site::delete(&mut context.pool(), site.id).await?; Ok(()) } diff --git a/crates/db_views/src/custom_emoji_view.rs b/crates/db_views/src/custom_emoji_view.rs index a346c086d..606e807e9 100644 --- a/crates/db_views/src/custom_emoji_view.rs +++ b/crates/db_views/src/custom_emoji_view.rs @@ -76,16 +76,16 @@ impl CustomEmojiView { fn from_tuple_to_vec(items: Vec) -> Vec { let mut result = Vec::new(); let mut hash: HashMap> = HashMap::new(); - for item in &items { - let emoji_id: CustomEmojiId = item.0.id; + for (emoji, keyword) in &items { + let emoji_id: CustomEmojiId = emoji.id; if let std::collections::hash_map::Entry::Vacant(e) = hash.entry(emoji_id) { e.insert(Vec::new()); result.push(CustomEmojiView { - custom_emoji: item.0.clone(), + custom_emoji: emoji.clone(), keywords: Vec::new(), }) } - if let Some(item_keyword) = &item.1 { + if let Some(item_keyword) = &keyword { if let Some(keywords) = hash.get_mut(&emoji_id) { keywords.push(item_keyword.clone()) } diff --git a/crates/utils/src/utils/markdown/image_links.rs b/crates/utils/src/utils/markdown/image_links.rs index a21bb6f41..7456190e4 100644 --- a/crates/utils/src/utils/markdown/image_links.rs +++ b/crates/utils/src/utils/markdown/image_links.rs @@ -42,13 +42,10 @@ pub fn markdown_rewrite_image_links(mut src: String) -> (String, Vec) { pub fn markdown_handle_title(src: &str, start: usize, end: usize) -> (&str, Option<&str>) { let content = src.get(start..end).unwrap_or_default(); // necessary for custom emojis which look like `![name](url "title")` - let (url, extra) = if content.contains(' ') { - let split = content.split_once(' ').expect("split is valid"); - (split.0, Some(split.1)) - } else { - (content, None) - }; - (url, extra) + match content.split_once(' ') { + Some((a, b)) => (a, Some(b)), + _ => (content, None), + } } pub fn markdown_find_links(src: &str) -> Vec<(usize, usize)> { @@ -61,9 +58,9 @@ fn find_urls(src: &str) -> Vec<(usize, usize)> { let mut links_offsets = vec![]; ast.walk(|node, _depth| { if let Some(image) = node.cast::() { - let node_offsets = node.srcmap.expect("srcmap is none").get_byte_offsets(); - let start_offset = node_offsets.1 - image.url_len() - 1 - image.title_len(); - let end_offset = node_offsets.1 - 1; + let (_, node_offset) = node.srcmap.expect("srcmap is none").get_byte_offsets(); + let start_offset = node_offset - image.url_len() - 1 - image.title_len(); + let end_offset = node_offset - 1; links_offsets.push((start_offset, end_offset)); } diff --git a/src/scheduled_tasks.rs b/src/scheduled_tasks.rs index 2f99fe8a1..75942d3dd 100644 --- a/src/scheduled_tasks.rs +++ b/src/scheduled_tasks.rs @@ -393,10 +393,10 @@ async fn active_counts(pool: &mut DbPool<'_>) { ("6 months", "half_year"), ]; - for i in &intervals { + for (full_form, abbr) in &intervals { let update_site_stmt = format!( "update site_aggregates set users_active_{} = (select * from site_aggregates_activity('{}')) where site_id = 1", - i.1, i.0 + abbr, full_form ); sql_query(update_site_stmt) .execute(&mut conn) @@ -404,7 +404,7 @@ async fn active_counts(pool: &mut DbPool<'_>) { .inspect_err(|e| error!("Failed to update site stats: {e}")) .ok(); - let update_community_stmt = format!("update community_aggregates ca set users_active_{} = mv.count_ from community_aggregates_activity('{}') mv where ca.community_id = mv.community_id_", i.1, i.0); + let update_community_stmt = format!("update community_aggregates ca set users_active_{} = mv.count_ from community_aggregates_activity('{}') mv where ca.community_id = mv.community_id_", abbr, full_form); sql_query(update_community_stmt) .execute(&mut conn) .await From e8875dec99456c4217a3e844eb67bdca756a02ff Mon Sep 17 00:00:00 2001 From: flamingos-cant <45780476+flamingo-cant-draw@users.noreply.github.com> Date: Thu, 31 Oct 2024 12:13:42 +0000 Subject: [PATCH 3/3] Append attachments to comments (#5143) * Append attachments to comments. * fmt * Proxy images + use newlines for separation * Use md for plain links * Use proxy_image_link directly --- crates/api_common/src/utils.rs | 2 +- .../objects/{note.json => note_1.json} | 0 .../apub/assets/mastodon/objects/note_2.json | 79 +++++++++++++++++++ crates/apub/src/objects/comment.rs | 4 +- crates/apub/src/objects/mod.rs | 19 ++++- crates/apub/src/protocol/objects/mod.rs | 3 +- crates/apub/src/protocol/objects/note.rs | 8 +- crates/apub/src/protocol/objects/page.rs | 21 ++++- 8 files changed, 130 insertions(+), 6 deletions(-) rename crates/apub/assets/mastodon/objects/{note.json => note_1.json} (100%) create mode 100644 crates/apub/assets/mastodon/objects/note_2.json diff --git a/crates/api_common/src/utils.rs b/crates/api_common/src/utils.rs index 87cdf9eef..e358d483b 100644 --- a/crates/api_common/src/utils.rs +++ b/crates/api_common/src/utils.rs @@ -1110,7 +1110,7 @@ async fn proxy_image_link_internal( /// Rewrite a link to go through `/api/v3/image_proxy` endpoint. This is only for remote urls and /// if image_proxy setting is enabled. -pub(crate) async fn proxy_image_link(link: Url, context: &LemmyContext) -> LemmyResult { +pub async fn proxy_image_link(link: Url, context: &LemmyContext) -> LemmyResult { proxy_image_link_internal( link, context.settings().pictrs_config()?.image_mode(), diff --git a/crates/apub/assets/mastodon/objects/note.json b/crates/apub/assets/mastodon/objects/note_1.json similarity index 100% rename from crates/apub/assets/mastodon/objects/note.json rename to crates/apub/assets/mastodon/objects/note_1.json diff --git a/crates/apub/assets/mastodon/objects/note_2.json b/crates/apub/assets/mastodon/objects/note_2.json new file mode 100644 index 000000000..b8c22b976 --- /dev/null +++ b/crates/apub/assets/mastodon/objects/note_2.json @@ -0,0 +1,79 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + { + "ostatus": "http://ostatus.org#", + "atomUri": "ostatus:atomUri", + "inReplyToAtomUri": "ostatus:inReplyToAtomUri", + "conversation": "ostatus:conversation", + "sensitive": "as:sensitive", + "toot": "http://joinmastodon.org/ns#", + "votersCount": "toot:votersCount", + "blurhash": "toot:blurhash", + "focalPoint": { + "@container": "@list", + "@id": "toot:focalPoint" + } + } + ], + "id": "https://floss.social/users/kde/statuses/113306831140126616", + "type": "Note", + "summary": null, + "inReplyTo": "https://floss.social/users/kde/statuses/113306824627995724", + "published": "2024-10-14T16:57:15Z", + "url": "https://floss.social/@kde/113306831140126616", + "attributedTo": "https://floss.social/users/kde", + "to": ["https://www.w3.org/ns/activitystreams#Public"], + "cc": [ + "https://floss.social/users/kde/followers", + "https://lemmy.kde.social/c/kde", + "https://lemmy.kde.social/c/kde/followers" + ], + "sensitive": false, + "atomUri": "https://floss.social/users/kde/statuses/113306831140126616", + "inReplyToAtomUri": "https://floss.social/users/kde/statuses/113306824627995724", + "conversation": "tag:floss.social,2024-10-14:objectId=71424279:objectType=Conversation", + "content": "

@kde@lemmy.kde.social

We also need funding 💶 to keep the gears turning! Please support us with a donation:

https://kde.org/donate/

[3/3]

", + "contentMap": { + "en": "

@kde@lemmy.kde.social

We also need funding 💶 to keep the gears turning! Please support us with a donation:

https://kde.org/donate/

[3/3]

" + }, + "attachment": [ + { + "type": "Document", + "mediaType": "image/jpeg", + "url": "https://cdn.masto.host/floss/media_attachments/files/113/306/826/682/985/891/original/c8d906a2f2ab2334.jpg", + "name": "The KDE dragons Katie and Konqi stand on either side of a pot filling up with gold coins. Donate!", + "blurhash": "USQv:h-W-qI-^,W;RPs=^-R%NZxbo#sDobSc", + "focalPoint": [0.0, 0.0], + "width": 1500, + "height": 1095 + } + ], + "tag": [ + { + "type": "Mention", + "href": "https://lemmy.kde.social/c/kde", + "name": "@kde@lemmy.kde.social" + } + ], + "replies": { + "id": "https://floss.social/users/kde/statuses/113306831140126616/replies", + "type": "Collection", + "first": { + "type": "CollectionPage", + "next": "https://floss.social/users/kde/statuses/113306831140126616/replies?only_other_accounts=true&page=true", + "partOf": "https://floss.social/users/kde/statuses/113306831140126616/replies", + "items": [] + } + }, + "likes": { + "id": "https://floss.social/users/kde/statuses/113306831140126616/likes", + "type": "Collection", + "totalItems": 39 + }, + "shares": { + "id": "https://floss.social/users/kde/statuses/113306831140126616/shares", + "type": "Collection", + "totalItems": 24 + } +} diff --git a/crates/apub/src/objects/comment.rs b/crates/apub/src/objects/comment.rs index e6cc2ba85..6e13afc91 100644 --- a/crates/apub/src/objects/comment.rs +++ b/crates/apub/src/objects/comment.rs @@ -3,7 +3,7 @@ use crate::{ check_apub_id_valid_with_strictness, fetcher::markdown_links::markdown_rewrite_remote_links, mentions::collect_non_local_mentions, - objects::{read_from_string_or_source, verify_is_remote_object}, + objects::{append_attachments_to_comment, read_from_string_or_source, verify_is_remote_object}, protocol::{ objects::{note::Note, LanguageTag}, InCommunity, @@ -124,6 +124,7 @@ impl Object for ApubComment { distinguished: Some(self.distinguished), language, audience: Some(community.actor_id.into()), + attachment: vec![], }; Ok(note) @@ -181,6 +182,7 @@ impl Object for ApubComment { let local_site = LocalSite::read(&mut context.pool()).await.ok(); let slur_regex = &local_site_opt_to_slur_regex(&local_site); let url_blocklist = get_url_blocklist(context).await?; + let content = append_attachments_to_comment(content, ¬e.attachment, context).await?; let content = process_markdown(&content, slur_regex, &url_blocklist, context).await?; let content = markdown_rewrite_remote_links(content, context).await; let language_id = Some( diff --git a/crates/apub/src/objects/mod.rs b/crates/apub/src/objects/mod.rs index e199ebfad..f837f7ad3 100644 --- a/crates/apub/src/objects/mod.rs +++ b/crates/apub/src/objects/mod.rs @@ -1,4 +1,4 @@ -use crate::protocol::Source; +use crate::protocol::{objects::page::Attachment, Source}; use activitypub_federation::{ config::Data, fetch::object_id::ObjectId, @@ -46,6 +46,23 @@ pub(crate) fn read_from_string_or_source_opt( .map(|content| read_from_string_or_source(content, media_type, source)) } +pub(crate) async fn append_attachments_to_comment( + content: String, + attachments: &[Attachment], + context: &Data, +) -> LemmyResult { + let mut content = content; + // Don't modify comments with no attachments + if !attachments.is_empty() { + content += "\n"; + for attachment in attachments { + content = content + "\n" + &attachment.as_markdown(context).await?; + } + } + + Ok(content) +} + /// When for example a Post is made in a remote community, the community will send it back, /// wrapped in Announce. If we simply receive this like any other federated object, overwrite the /// existing, local Post. In particular, it will set the field local = false, so that the object diff --git a/crates/apub/src/protocol/objects/mod.rs b/crates/apub/src/protocol/objects/mod.rs index dbba1bb8a..00fe26d2b 100644 --- a/crates/apub/src/protocol/objects/mod.rs +++ b/crates/apub/src/protocol/objects/mod.rs @@ -145,7 +145,8 @@ mod tests { #[test] fn test_parse_objects_mastodon() -> LemmyResult<()> { test_json::("assets/mastodon/objects/person.json")?; - test_json::("assets/mastodon/objects/note.json")?; + test_json::("assets/mastodon/objects/note_1.json")?; + test_json::("assets/mastodon/objects/note_2.json")?; test_json::("assets/mastodon/objects/page.json")?; Ok(()) } diff --git a/crates/apub/src/protocol/objects/note.rs b/crates/apub/src/protocol/objects/note.rs index e3e204254..21b5220f5 100644 --- a/crates/apub/src/protocol/objects/note.rs +++ b/crates/apub/src/protocol/objects/note.rs @@ -3,7 +3,11 @@ use crate::{ fetcher::post_or_comment::PostOrComment, mentions::MentionOrValue, objects::{comment::ApubComment, community::ApubCommunity, person::ApubPerson, post::ApubPost}, - protocol::{objects::LanguageTag, InCommunity, Source}, + protocol::{ + objects::{page::Attachment, LanguageTag}, + InCommunity, + Source, + }, }; use activitypub_federation::{ config::Data, @@ -50,6 +54,8 @@ pub struct Note { pub(crate) distinguished: Option, pub(crate) language: Option, pub(crate) audience: Option>, + #[serde(default)] + pub(crate) attachment: Vec, } impl Note { diff --git a/crates/apub/src/protocol/objects/page.rs b/crates/apub/src/protocol/objects/page.rs index 97f767573..3ce720bc0 100644 --- a/crates/apub/src/protocol/objects/page.rs +++ b/crates/apub/src/protocol/objects/page.rs @@ -19,7 +19,7 @@ use activitypub_federation::{ }; use chrono::{DateTime, Utc}; use itertools::Itertools; -use lemmy_api_common::context::LemmyContext; +use lemmy_api_common::{context::LemmyContext, utils::proxy_image_link}; use lemmy_utils::error::{FederationError, LemmyError, LemmyErrorType, LemmyResult}; use serde::{de::Error, Deserialize, Deserializer, Serialize}; use serde_with::skip_serializing_none; @@ -93,6 +93,7 @@ pub(crate) struct Document { #[serde(rename = "type")] kind: DocumentType, url: Url, + media_type: Option, /// Used for alt_text name: Option, } @@ -124,6 +125,24 @@ impl Attachment { _ => None, } } + + pub(crate) async fn as_markdown(&self, context: &Data) -> LemmyResult { + let (url, name, media_type) = match self { + Attachment::Image(i) => (i.url.clone(), i.name.clone(), Some(String::from("image"))), + Attachment::Document(d) => (d.url.clone(), d.name.clone(), d.media_type.clone()), + Attachment::Link(l) => (l.href.clone(), None, l.media_type.clone()), + }; + + let is_image = + media_type.is_some_and(|media| media.starts_with("video") || media.starts_with("image")); + + if is_image { + let url = proxy_image_link(url, context).await?; + Ok(format!("![{}]({url})", name.unwrap_or_default())) + } else { + Ok(format!("[{url}]({url})")) + } + } } #[derive(Clone, Debug, Deserialize, Serialize)]