From 265ad9b0600ecb474deafc91a8d1d5585acf6ba0 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Wed, 10 Jun 2020 14:30:41 -0400 Subject: [PATCH] Parse out in reply to field. Fixes #694 - When a comment or post doesn't exist locally, yet we receive an activitypub with it (for example, a nested comment update, for a community we just subscribed to, also with a post we don't have...), then fetch it. --- server/src/apub/comment.rs | 19 ++-- server/src/apub/fetcher.rs | 35 +++++++ server/src/apub/shared_inbox.rs | 39 ++++---- ui/src/api_tests/api.spec.ts | 172 +++++++++++++++++++++++++++++++- 4 files changed, 237 insertions(+), 28 deletions(-) diff --git a/server/src/apub/comment.rs b/server/src/apub/comment.rs index d18908d0e..0a513f332 100644 --- a/server/src/apub/comment.rs +++ b/server/src/apub/comment.rs @@ -5,7 +5,11 @@ use crate::{ create_apub_tombstone_response, create_tombstone, fetch_webfinger_url, - fetcher::get_or_fetch_and_upsert_remote_user, + fetcher::{ + get_or_fetch_and_insert_remote_comment, + get_or_fetch_and_insert_remote_post, + get_or_fetch_and_upsert_remote_user, + }, ActorType, ApubLikeableType, ApubObjectType, @@ -115,22 +119,21 @@ impl FromApub for CommentForm { let mut in_reply_tos = oprops.get_many_in_reply_to_xsd_any_uris().unwrap(); let post_ap_id = in_reply_tos.next().unwrap().to_string(); + // This post, or the parent comment might not yet exist on this server yet, fetch them. + let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?; + // The 2nd item, if it exists, is the parent comment apub_id + // For deeply nested comments, FromApub automatically gets called recursively let parent_id: Option = match in_reply_tos.next() { Some(parent_comment_uri) => { - let parent_comment_uri_str = &parent_comment_uri.to_string(); - let parent_comment = Comment::read_from_apub_id(&conn, &parent_comment_uri_str)?; + let parent_comment_ap_id = &parent_comment_uri.to_string(); + let parent_comment = get_or_fetch_and_insert_remote_comment(&parent_comment_ap_id, &conn)?; Some(parent_comment.id) } None => None, }; - // TODO this failed because a mention on a post that wasn't on this server yet. Has to do with - // fetching replytos - dbg!(&post_ap_id); - let post = Post::read_from_apub_id(&conn, &post_ap_id)?; - Ok(CommentForm { creator_id: creator.id, post_id: post.id, diff --git a/server/src/apub/fetcher.rs b/server/src/apub/fetcher.rs index c7467c5b9..05b7492ee 100644 --- a/server/src/apub/fetcher.rs +++ b/server/src/apub/fetcher.rs @@ -254,6 +254,22 @@ fn upsert_post(post_form: &PostForm, conn: &PgConnection) -> Result } } +pub fn get_or_fetch_and_insert_remote_post( + post_ap_id: &str, + conn: &PgConnection, +) -> Result { + match Post::read_from_apub_id(conn, post_ap_id) { + Ok(p) => Ok(p), + Err(NotFound {}) => { + debug!("Fetching and creating remote post: {}", post_ap_id); + let post = fetch_remote_object::(&Url::parse(post_ap_id)?)?; + let post_form = PostForm::from_apub(&post, conn)?; + Ok(Post::create(conn, &post_form)?) + } + Err(e) => Err(Error::from(e)), + } +} + fn upsert_comment(comment_form: &CommentForm, conn: &PgConnection) -> Result { let existing = Comment::read_from_apub_id(conn, &comment_form.ap_id); match existing { @@ -263,6 +279,25 @@ fn upsert_comment(comment_form: &CommentForm, conn: &PgConnection) -> Result Result { + match Comment::read_from_apub_id(conn, comment_ap_id) { + Ok(p) => Ok(p), + Err(NotFound {}) => { + debug!( + "Fetching and creating remote comment and its parents: {}", + comment_ap_id + ); + let comment = fetch_remote_object::(&Url::parse(comment_ap_id)?)?; + let comment_form = CommentForm::from_apub(&comment, conn)?; + Ok(Comment::create(conn, &comment_form)?) + } + Err(e) => Err(Error::from(e)), + } +} + // TODO It should not be fetching data from a community outbox. // All posts, comments, comment likes, etc should be posts to our community_inbox // The only data we should be periodically fetching (if it hasn't been fetched in the last day diff --git a/server/src/apub/shared_inbox.rs b/server/src/apub/shared_inbox.rs index 3ecc873e5..1ada6ad1d 100644 --- a/server/src/apub/shared_inbox.rs +++ b/server/src/apub/shared_inbox.rs @@ -6,7 +6,12 @@ use crate::{ }, apub::{ extensions::signatures::verify, - fetcher::{get_or_fetch_and_upsert_remote_community, get_or_fetch_and_upsert_remote_user}, + fetcher::{ + get_or_fetch_and_insert_remote_comment, + get_or_fetch_and_insert_remote_post, + get_or_fetch_and_upsert_remote_community, + get_or_fetch_and_upsert_remote_user, + }, FromApub, GroupExt, PageExt, @@ -427,7 +432,7 @@ fn receive_update_post( insert_activity(&conn, user.id, &update, false)?; let post = PostForm::from_apub(&page, conn)?; - let post_id = Post::read_from_apub_id(conn, &post.ap_id)?.id; + let post_id = get_or_fetch_and_insert_remote_post(&post.ap_id, &conn)?.id; Post::update(conn, post_id, &post)?; // Refetch the view @@ -465,7 +470,7 @@ fn receive_like_post( insert_activity(&conn, user.id, &like, false)?; let post = PostForm::from_apub(&page, conn)?; - let post_id = Post::read_from_apub_id(conn, &post.ap_id)?.id; + let post_id = get_or_fetch_and_insert_remote_post(&post.ap_id, &conn)?.id; let like_form = PostLikeForm { post_id, @@ -514,7 +519,7 @@ fn receive_dislike_post( insert_activity(&conn, user.id, &dislike, false)?; let post = PostForm::from_apub(&page, conn)?; - let post_id = Post::read_from_apub_id(conn, &post.ap_id)?.id; + let post_id = get_or_fetch_and_insert_remote_post(&post.ap_id, &conn)?.id; let like_form = PostLikeForm { post_id, @@ -563,7 +568,7 @@ fn receive_update_comment( insert_activity(&conn, user.id, &update, false)?; let comment = CommentForm::from_apub(¬e, &conn)?; - let comment_id = Comment::read_from_apub_id(conn, &comment.ap_id)?.id; + let comment_id = get_or_fetch_and_insert_remote_comment(&comment.ap_id, &conn)?.id; let updated_comment = Comment::update(conn, comment_id, &comment)?; let post = Post::read(&conn, updated_comment.post_id)?; @@ -608,7 +613,7 @@ fn receive_like_comment( insert_activity(&conn, user.id, &like, false)?; let comment = CommentForm::from_apub(¬e, &conn)?; - let comment_id = Comment::read_from_apub_id(conn, &comment.ap_id)?.id; + let comment_id = get_or_fetch_and_insert_remote_comment(&comment.ap_id, &conn)?.id; let like_form = CommentLikeForm { comment_id, post_id: comment.post_id, @@ -662,7 +667,7 @@ fn receive_dislike_comment( insert_activity(&conn, user.id, &dislike, false)?; let comment = CommentForm::from_apub(¬e, &conn)?; - let comment_id = Comment::read_from_apub_id(conn, &comment.ap_id)?.id; + let comment_id = get_or_fetch_and_insert_remote_comment(&comment.ap_id, &conn)?.id; let like_form = CommentLikeForm { comment_id, post_id: comment.post_id, @@ -838,7 +843,7 @@ fn receive_delete_post( insert_activity(&conn, user.id, &delete, false)?; let post_ap_id = PostForm::from_apub(&page, conn)?.ap_id; - let post = Post::read_from_apub_id(conn, &post_ap_id)?; + let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?; let post_form = PostForm { name: post.name.to_owned(), @@ -901,7 +906,7 @@ fn receive_remove_post( insert_activity(&conn, mod_.id, &remove, false)?; let post_ap_id = PostForm::from_apub(&page, conn)?.ap_id; - let post = Post::read_from_apub_id(conn, &post_ap_id)?; + let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?; let post_form = PostForm { name: post.name.to_owned(), @@ -964,7 +969,7 @@ fn receive_delete_comment( insert_activity(&conn, user.id, &delete, false)?; let comment_ap_id = CommentForm::from_apub(¬e, &conn)?.ap_id; - let comment = Comment::read_from_apub_id(conn, &comment_ap_id)?; + let comment = get_or_fetch_and_insert_remote_comment(&comment_ap_id, &conn)?; let comment_form = CommentForm { content: comment.content.to_owned(), parent_id: comment.parent_id, @@ -1024,7 +1029,7 @@ fn receive_remove_comment( insert_activity(&conn, mod_.id, &remove, false)?; let comment_ap_id = CommentForm::from_apub(¬e, &conn)?.ap_id; - let comment = Comment::read_from_apub_id(conn, &comment_ap_id)?; + let comment = get_or_fetch_and_insert_remote_comment(&comment_ap_id, &conn)?; let comment_form = CommentForm { content: comment.content.to_owned(), parent_id: comment.parent_id, @@ -1144,7 +1149,7 @@ fn receive_undo_delete_comment( insert_activity(&conn, user.id, &delete, false)?; let comment_ap_id = CommentForm::from_apub(¬e, &conn)?.ap_id; - let comment = Comment::read_from_apub_id(conn, &comment_ap_id)?; + let comment = get_or_fetch_and_insert_remote_comment(&comment_ap_id, &conn)?; let comment_form = CommentForm { content: comment.content.to_owned(), parent_id: comment.parent_id, @@ -1204,7 +1209,7 @@ fn receive_undo_remove_comment( insert_activity(&conn, mod_.id, &remove, false)?; let comment_ap_id = CommentForm::from_apub(¬e, &conn)?.ap_id; - let comment = Comment::read_from_apub_id(conn, &comment_ap_id)?; + let comment = get_or_fetch_and_insert_remote_comment(&comment_ap_id, &conn)?; let comment_form = CommentForm { content: comment.content.to_owned(), parent_id: comment.parent_id, @@ -1264,7 +1269,7 @@ fn receive_undo_delete_post( insert_activity(&conn, user.id, &delete, false)?; let post_ap_id = PostForm::from_apub(&page, conn)?.ap_id; - let post = Post::read_from_apub_id(conn, &post_ap_id)?; + let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?; let post_form = PostForm { name: post.name.to_owned(), @@ -1327,7 +1332,7 @@ fn receive_undo_remove_post( insert_activity(&conn, mod_.id, &remove, false)?; let post_ap_id = PostForm::from_apub(&page, conn)?.ap_id; - let post = Post::read_from_apub_id(conn, &post_ap_id)?; + let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?; let post_form = PostForm { name: post.name.to_owned(), @@ -1537,7 +1542,7 @@ fn receive_undo_like_comment( insert_activity(&conn, user.id, &like, false)?; let comment = CommentForm::from_apub(¬e, &conn)?; - let comment_id = Comment::read_from_apub_id(conn, &comment.ap_id)?.id; + let comment_id = get_or_fetch_and_insert_remote_comment(&comment.ap_id, &conn)?.id; let like_form = CommentLikeForm { comment_id, post_id: comment.post_id, @@ -1586,7 +1591,7 @@ fn receive_undo_like_post( insert_activity(&conn, user.id, &like, false)?; let post = PostForm::from_apub(&page, conn)?; - let post_id = Post::read_from_apub_id(conn, &post.ap_id)?.id; + let post_id = get_or_fetch_and_insert_remote_post(&post.ap_id, &conn)?.id; let like_form = PostLikeForm { post_id, diff --git a/ui/src/api_tests/api.spec.ts b/ui/src/api_tests/api.spec.ts index 4e38cfee4..7337201c7 100644 --- a/ui/src/api_tests/api.spec.ts +++ b/ui/src/api_tests/api.spec.ts @@ -101,7 +101,7 @@ describe('main', () => { nsfw: false, }; - let createResponse: PostResponse = await fetch( + let createPostRes: PostResponse = await fetch( `${lemmyAlphaApiUrl}/post`, { method: 'POST', @@ -111,9 +111,9 @@ describe('main', () => { body: wrapper(postForm), } ).then(d => d.json()); - expect(createResponse.post.name).toBe(name); + expect(createPostRes.post.name).toBe(name); - let searchUrl = `${lemmyBetaApiUrl}/search?q=${createResponse.post.ap_id}&type_=All&sort=TopAll`; + let searchUrl = `${lemmyBetaApiUrl}/search?q=${createPostRes.post.ap_id}&type_=All&sort=TopAll`; let searchResponse: SearchResponse = await fetch(searchUrl, { method: 'GET', }).then(d => d.json()); @@ -1314,6 +1314,172 @@ describe('main', () => { expect(getPostRes.comments[0].score).toBe(1); }); }); + + describe('fetch inreplytos', () => { + test('A is unsubbed from B, B makes a post, and some embedded comments, A subs to B, B updates the lowest level comment, A fetches both the post and all the inreplyto comments for that post.', async () => { + // Check that A is subscribed to B + let followedCommunitiesUrl = `${lemmyAlphaApiUrl}/user/followed_communities?&auth=${lemmyAlphaAuth}`; + let followedCommunitiesRes: GetFollowedCommunitiesResponse = await fetch( + followedCommunitiesUrl, + { + method: 'GET', + } + ).then(d => d.json()); + expect(followedCommunitiesRes.communities[1].community_local).toBe(false); + + // A unsubs from B (communities ids 3-5) + for (let i = 3; i <= 5; i++) { + let unfollowForm: FollowCommunityForm = { + community_id: i, + follow: false, + auth: lemmyAlphaAuth, + }; + + let unfollowRes: CommunityResponse = await fetch( + `${lemmyAlphaApiUrl}/community/follow`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: wrapper(unfollowForm), + } + ).then(d => d.json()); + expect(unfollowRes.community.local).toBe(false); + } + + // Check that you are unsubscribed from all of them locally + let followedCommunitiesResAgain: GetFollowedCommunitiesResponse = await fetch( + followedCommunitiesUrl, + { + method: 'GET', + } + ).then(d => d.json()); + expect(followedCommunitiesResAgain.communities.length).toBe(1); + + // B creates a post, and two comments, should be invisible to A + let betaPostName = 'Test post on B, invisible to A at first'; + let postForm: PostForm = { + name: betaPostName, + auth: lemmyBetaAuth, + community_id: 2, + creator_id: 2, + nsfw: false, + }; + + let createPostRes: PostResponse = await fetch(`${lemmyBetaApiUrl}/post`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: wrapper(postForm), + }).then(d => d.json()); + expect(createPostRes.post.name).toBe(betaPostName); + + // B creates a comment, then a child one of that. + let parentCommentContent = 'An invisible top level comment from beta'; + let createParentCommentForm: CommentForm = { + content: parentCommentContent, + post_id: createPostRes.post.id, + auth: lemmyBetaAuth, + }; + + let createParentCommentRes: CommentResponse = await fetch( + `${lemmyBetaApiUrl}/comment`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: wrapper(createParentCommentForm), + } + ).then(d => d.json()); + expect(createParentCommentRes.comment.content).toBe(parentCommentContent); + + let childCommentContent = 'An invisible child comment from beta'; + let createChildCommentForm: CommentForm = { + content: childCommentContent, + parent_id: createParentCommentRes.comment.id, + post_id: createPostRes.post.id, + auth: lemmyBetaAuth, + }; + + let createChildCommentRes: CommentResponse = await fetch( + `${lemmyBetaApiUrl}/comment`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: wrapper(createChildCommentForm), + } + ).then(d => d.json()); + expect(createChildCommentRes.comment.content).toBe(childCommentContent); + + // Follow again, for other tests + let searchUrl = `${lemmyAlphaApiUrl}/search?q=!main@lemmy_beta:8550&type_=All&sort=TopAll`; + + let searchResponse: SearchResponse = await fetch(searchUrl, { + method: 'GET', + }).then(d => d.json()); + + expect(searchResponse.communities[0].name).toBe('main'); + + let followForm: FollowCommunityForm = { + community_id: searchResponse.communities[0].id, + follow: true, + auth: lemmyAlphaAuth, + }; + + let followResAgain: CommunityResponse = await fetch( + `${lemmyAlphaApiUrl}/community/follow`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: wrapper(followForm), + } + ).then(d => d.json()); + + // Make sure the follow response went through + expect(followResAgain.community.local).toBe(false); + expect(followResAgain.community.name).toBe('main'); + + let updatedCommentContent = 'An update child comment from beta'; + let updatedCommentForm: CommentForm = { + content: updatedCommentContent, + post_id: createPostRes.post.id, + edit_id: createChildCommentRes.comment.id, + auth: lemmyBetaAuth, + creator_id: 2, + }; + + let updateResponse: CommentResponse = await fetch( + `${lemmyBetaApiUrl}/comment`, + { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + }, + body: wrapper(updatedCommentForm), + } + ).then(d => d.json()); + expect(updateResponse.comment.content).toBe(updatedCommentContent); + + // Make sure that A picked up the post, parent comment, and child comment + let getPostUrl = `${lemmyAlphaApiUrl}/post?id=6`; + let getPostRes: GetPostResponse = await fetch(getPostUrl, { + method: 'GET', + }).then(d => d.json()); + + expect(getPostRes.post.name).toBe(betaPostName); + expect(getPostRes.comments[1].content).toBe(parentCommentContent); + expect(getPostRes.comments[0].content).toBe(updatedCommentContent); + expect(getPostRes.post.community_local).toBe(false); + expect(getPostRes.post.creator_local).toBe(false); + }); + }); }); function wrapper(form: any): string {