LemmyNet
/
lemmy
3
2
Fork 3
Code Pull Requests 1 Releases Wiki Activity

Parse out in reply to field. Fixes #694 (#799) 

- When a comment or post doesn't exist locally, yet we receive
  an activitypub with it (for example, a nested comment update,
  for a community we just subscribed to, also with a post we
  don't have...), then fetch it.
This commit is contained in:
Dessalines 2020-06-11 11:16:33 -04:00 committed by GitHub
parent 0f1a8ec928
commit cfa40e482a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 237 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
server/src/apub/comment.rs
View File

@ -5,7 +5,11 @@ use crate::{
create_apub_tombstone_response,
create_tombstone,
fetch_webfinger_url,
fetcher::get_or_fetch_and_upsert_remote_user,
fetcher::{
get_or_fetch_and_insert_remote_comment,
get_or_fetch_and_insert_remote_post,
get_or_fetch_and_upsert_remote_user,
},
ActorType,
ApubLikeableType,
ApubObjectType,
@ -115,22 +119,21 @@ impl FromApub for CommentForm {
let mut in_reply_tos = oprops.get_many_in_reply_to_xsd_any_uris().unwrap();
let post_ap_id = in_reply_tos.next().unwrap().to_string();
// This post, or the parent comment might not yet exist on this server yet, fetch them.
let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?;
// The 2nd item, if it exists, is the parent comment apub_id
// For deeply nested comments, FromApub automatically gets called recursively
let parent_id: Option<i32> = match in_reply_tos.next() {
Some(parent_comment_uri) => {
let parent_comment_uri_str = &parent_comment_uri.to_string();
let parent_comment = Comment::read_from_apub_id(&conn, &parent_comment_uri_str)?;
let parent_comment_ap_id = &parent_comment_uri.to_string();
let parent_comment = get_or_fetch_and_insert_remote_comment(&parent_comment_ap_id, &conn)?;
Some(parent_comment.id)
}
None => None,
};
// TODO this failed because a mention on a post that wasn't on this server yet. Has to do with
// fetching replytos
dbg!(&post_ap_id);
let post = Post::read_from_apub_id(&conn, &post_ap_id)?;
Ok(CommentForm {
creator_id: creator.id,
post_id: post.id,

35
server/src/apub/fetcher.rs
View File

@ -254,6 +254,22 @@ fn upsert_post(post_form: &PostForm, conn: &PgConnection) -> Result<Post, Error>
}
}
pub fn get_or_fetch_and_insert_remote_post(
post_ap_id: &str,
conn: &PgConnection,
) -> Result<Post, Error> {
match Post::read_from_apub_id(conn, post_ap_id) {
Ok(p) => Ok(p),
Err(NotFound {}) => {
debug!("Fetching and creating remote post: {}", post_ap_id);
let post = fetch_remote_object::<PageExt>(&Url::parse(post_ap_id)?)?;
let post_form = PostForm::from_apub(&post, conn)?;
Ok(Post::create(conn, &post_form)?)
}
Err(e) => Err(Error::from(e)),
}
}
fn upsert_comment(comment_form: &CommentForm, conn: &PgConnection) -> Result<Comment, Error> {
let existing = Comment::read_from_apub_id(conn, &comment_form.ap_id);
match existing {
@ -263,6 +279,25 @@ fn upsert_comment(comment_form: &CommentForm, conn: &PgConnection) -> Result<Com
}
}
pub fn get_or_fetch_and_insert_remote_comment(
comment_ap_id: &str,
conn: &PgConnection,
) -> Result<Comment, Error> {
match Comment::read_from_apub_id(conn, comment_ap_id) {
Ok(p) => Ok(p),
Err(NotFound {}) => {
debug!(
"Fetching and creating remote comment and its parents: {}",
comment_ap_id
);
let comment = fetch_remote_object::<Note>(&Url::parse(comment_ap_id)?)?;
let comment_form = CommentForm::from_apub(&comment, conn)?;
Ok(Comment::create(conn, &comment_form)?)
}
Err(e) => Err(Error::from(e)),
}
}
// TODO It should not be fetching data from a community outbox.
// All posts, comments, comment likes, etc should be posts to our community_inbox
// The only data we should be periodically fetching (if it hasn't been fetched in the last day

39
server/src/apub/shared_inbox.rs
View File

@ -6,7 +6,12 @@ use crate::{
},
apub::{
extensions::signatures::verify,
fetcher::{get_or_fetch_and_upsert_remote_community, get_or_fetch_and_upsert_remote_user},
fetcher::{
get_or_fetch_and_insert_remote_comment,
get_or_fetch_and_insert_remote_post,
get_or_fetch_and_upsert_remote_community,
get_or_fetch_and_upsert_remote_user,
},
FromApub,
GroupExt,
PageExt,
@ -427,7 +432,7 @@ fn receive_update_post(
insert_activity(&conn, user.id, &update, false)?;
let post = PostForm::from_apub(&page, conn)?;
let post_id = Post::read_from_apub_id(conn, &post.ap_id)?.id;
let post_id = get_or_fetch_and_insert_remote_post(&post.ap_id, &conn)?.id;
Post::update(conn, post_id, &post)?;
// Refetch the view
@ -465,7 +470,7 @@ fn receive_like_post(
insert_activity(&conn, user.id, &like, false)?;
let post = PostForm::from_apub(&page, conn)?;
let post_id = Post::read_from_apub_id(conn, &post.ap_id)?.id;
let post_id = get_or_fetch_and_insert_remote_post(&post.ap_id, &conn)?.id;
let like_form = PostLikeForm {
post_id,
@ -514,7 +519,7 @@ fn receive_dislike_post(
insert_activity(&conn, user.id, &dislike, false)?;
let post = PostForm::from_apub(&page, conn)?;
let post_id = Post::read_from_apub_id(conn, &post.ap_id)?.id;
let post_id = get_or_fetch_and_insert_remote_post(&post.ap_id, &conn)?.id;
let like_form = PostLikeForm {
post_id,
@ -563,7 +568,7 @@ fn receive_update_comment(
insert_activity(&conn, user.id, &update, false)?;
let comment = CommentForm::from_apub(&note, &conn)?;
let comment_id = Comment::read_from_apub_id(conn, &comment.ap_id)?.id;
let comment_id = get_or_fetch_and_insert_remote_comment(&comment.ap_id, &conn)?.id;
let updated_comment = Comment::update(conn, comment_id, &comment)?;
let post = Post::read(&conn, updated_comment.post_id)?;
@ -608,7 +613,7 @@ fn receive_like_comment(
insert_activity(&conn, user.id, &like, false)?;
let comment = CommentForm::from_apub(&note, &conn)?;
let comment_id = Comment::read_from_apub_id(conn, &comment.ap_id)?.id;
let comment_id = get_or_fetch_and_insert_remote_comment(&comment.ap_id, &conn)?.id;
let like_form = CommentLikeForm {
comment_id,
post_id: comment.post_id,
@ -662,7 +667,7 @@ fn receive_dislike_comment(
insert_activity(&conn, user.id, &dislike, false)?;
let comment = CommentForm::from_apub(&note, &conn)?;
let comment_id = Comment::read_from_apub_id(conn, &comment.ap_id)?.id;
let comment_id = get_or_fetch_and_insert_remote_comment(&comment.ap_id, &conn)?.id;
let like_form = CommentLikeForm {
comment_id,
post_id: comment.post_id,
@ -838,7 +843,7 @@ fn receive_delete_post(
insert_activity(&conn, user.id, &delete, false)?;
let post_ap_id = PostForm::from_apub(&page, conn)?.ap_id;
let post = Post::read_from_apub_id(conn, &post_ap_id)?;
let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?;
let post_form = PostForm {
name: post.name.to_owned(),
@ -901,7 +906,7 @@ fn receive_remove_post(
insert_activity(&conn, mod_.id, &remove, false)?;
let post_ap_id = PostForm::from_apub(&page, conn)?.ap_id;
let post = Post::read_from_apub_id(conn, &post_ap_id)?;
let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?;
let post_form = PostForm {
name: post.name.to_owned(),
@ -964,7 +969,7 @@ fn receive_delete_comment(
insert_activity(&conn, user.id, &delete, false)?;
let comment_ap_id = CommentForm::from_apub(&note, &conn)?.ap_id;
let comment = Comment::read_from_apub_id(conn, &comment_ap_id)?;
let comment = get_or_fetch_and_insert_remote_comment(&comment_ap_id, &conn)?;
let comment_form = CommentForm {
content: comment.content.to_owned(),
parent_id: comment.parent_id,
@ -1024,7 +1029,7 @@ fn receive_remove_comment(
insert_activity(&conn, mod_.id, &remove, false)?;
let comment_ap_id = CommentForm::from_apub(&note, &conn)?.ap_id;
let comment = Comment::read_from_apub_id(conn, &comment_ap_id)?;
let comment = get_or_fetch_and_insert_remote_comment(&comment_ap_id, &conn)?;
let comment_form = CommentForm {
content: comment.content.to_owned(),
parent_id: comment.parent_id,
@ -1144,7 +1149,7 @@ fn receive_undo_delete_comment(
insert_activity(&conn, user.id, &delete, false)?;
let comment_ap_id = CommentForm::from_apub(&note, &conn)?.ap_id;
let comment = Comment::read_from_apub_id(conn, &comment_ap_id)?;
let comment = get_or_fetch_and_insert_remote_comment(&comment_ap_id, &conn)?;
let comment_form = CommentForm {
content: comment.content.to_owned(),
parent_id: comment.parent_id,
@ -1204,7 +1209,7 @@ fn receive_undo_remove_comment(
insert_activity(&conn, mod_.id, &remove, false)?;
let comment_ap_id = CommentForm::from_apub(&note, &conn)?.ap_id;
let comment = Comment::read_from_apub_id(conn, &comment_ap_id)?;
let comment = get_or_fetch_and_insert_remote_comment(&comment_ap_id, &conn)?;
let comment_form = CommentForm {
content: comment.content.to_owned(),
parent_id: comment.parent_id,
@ -1264,7 +1269,7 @@ fn receive_undo_delete_post(
insert_activity(&conn, user.id, &delete, false)?;
let post_ap_id = PostForm::from_apub(&page, conn)?.ap_id;
let post = Post::read_from_apub_id(conn, &post_ap_id)?;
let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?;
let post_form = PostForm {
name: post.name.to_owned(),
@ -1327,7 +1332,7 @@ fn receive_undo_remove_post(
insert_activity(&conn, mod_.id, &remove, false)?;
let post_ap_id = PostForm::from_apub(&page, conn)?.ap_id;
let post = Post::read_from_apub_id(conn, &post_ap_id)?;
let post = get_or_fetch_and_insert_remote_post(&post_ap_id, &conn)?;
let post_form = PostForm {
name: post.name.to_owned(),
@ -1537,7 +1542,7 @@ fn receive_undo_like_comment(
insert_activity(&conn, user.id, &like, false)?;
let comment = CommentForm::from_apub(&note, &conn)?;
let comment_id = Comment::read_from_apub_id(conn, &comment.ap_id)?.id;
let comment_id = get_or_fetch_and_insert_remote_comment(&comment.ap_id, &conn)?.id;
let like_form = CommentLikeForm {
comment_id,
post_id: comment.post_id,
@ -1586,7 +1591,7 @@ fn receive_undo_like_post(
insert_activity(&conn, user.id, &like, false)?;
let post = PostForm::from_apub(&page, conn)?;
let post_id = Post::read_from_apub_id(conn, &post.ap_id)?.id;
let post_id = get_or_fetch_and_insert_remote_post(&post.ap_id, &conn)?.id;
let like_form = PostLikeForm {
post_id,

172
ui/src/api_tests/api.spec.ts vendored
View File

@ -101,7 +101,7 @@ describe('main', () => {
nsfw: false,
};
let createResponse: PostResponse = await fetch(
let createPostRes: PostResponse = await fetch(
`${lemmyAlphaApiUrl}/post`,
{
method: 'POST',
@ -111,9 +111,9 @@ describe('main', () => {
body: wrapper(postForm),
}
).then(d => d.json());
expect(createResponse.post.name).toBe(name);
expect(createPostRes.post.name).toBe(name);
let searchUrl = `${lemmyBetaApiUrl}/search?q=${createResponse.post.ap_id}&type_=All&sort=TopAll`;
let searchUrl = `${lemmyBetaApiUrl}/search?q=${createPostRes.post.ap_id}&type_=All&sort=TopAll`;
let searchResponse: SearchResponse = await fetch(searchUrl, {
method: 'GET',
}).then(d => d.json());
@ -1314,6 +1314,172 @@ describe('main', () => {
expect(getPostRes.comments[0].score).toBe(1);
});
});
describe('fetch inreplytos', () => {
test('A is unsubbed from B, B makes a post, and some embedded comments, A subs to B, B updates the lowest level comment, A fetches both the post and all the inreplyto comments for that post.', async () => {
// Check that A is subscribed to B
let followedCommunitiesUrl = `${lemmyAlphaApiUrl}/user/followed_communities?&auth=${lemmyAlphaAuth}`;
let followedCommunitiesRes: GetFollowedCommunitiesResponse = await fetch(
followedCommunitiesUrl,
{
method: 'GET',
}
).then(d => d.json());
expect(followedCommunitiesRes.communities[1].community_local).toBe(false);
// A unsubs from B (communities ids 3-5)
for (let i = 3; i <= 5; i++) {
let unfollowForm: FollowCommunityForm = {
community_id: i,
follow: false,
auth: lemmyAlphaAuth,
};
let unfollowRes: CommunityResponse = await fetch(
`${lemmyAlphaApiUrl}/community/follow`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: wrapper(unfollowForm),
}
).then(d => d.json());
expect(unfollowRes.community.local).toBe(false);
}
// Check that you are unsubscribed from all of them locally
let followedCommunitiesResAgain: GetFollowedCommunitiesResponse = await fetch(
followedCommunitiesUrl,
{
method: 'GET',
}
).then(d => d.json());
expect(followedCommunitiesResAgain.communities.length).toBe(1);
// B creates a post, and two comments, should be invisible to A
let betaPostName = 'Test post on B, invisible to A at first';
let postForm: PostForm = {
name: betaPostName,
auth: lemmyBetaAuth,
community_id: 2,
creator_id: 2,
nsfw: false,
};
let createPostRes: PostResponse = await fetch(`${lemmyBetaApiUrl}/post`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: wrapper(postForm),
}).then(d => d.json());
expect(createPostRes.post.name).toBe(betaPostName);
// B creates a comment, then a child one of that.
let parentCommentContent = 'An invisible top level comment from beta';
let createParentCommentForm: CommentForm = {
content: parentCommentContent,
post_id: createPostRes.post.id,
auth: lemmyBetaAuth,
};
let createParentCommentRes: CommentResponse = await fetch(
`${lemmyBetaApiUrl}/comment`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: wrapper(createParentCommentForm),
}
).then(d => d.json());
expect(createParentCommentRes.comment.content).toBe(parentCommentContent);
let childCommentContent = 'An invisible child comment from beta';
let createChildCommentForm: CommentForm = {
content: childCommentContent,
parent_id: createParentCommentRes.comment.id,
post_id: createPostRes.post.id,
auth: lemmyBetaAuth,
};
let createChildCommentRes: CommentResponse = await fetch(
`${lemmyBetaApiUrl}/comment`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: wrapper(createChildCommentForm),
}
).then(d => d.json());
expect(createChildCommentRes.comment.content).toBe(childCommentContent);
// Follow again, for other tests
let searchUrl = `${lemmyAlphaApiUrl}/search?q=!main@lemmy_beta:8550&type_=All&sort=TopAll`;
let searchResponse: SearchResponse = await fetch(searchUrl, {
method: 'GET',
}).then(d => d.json());
expect(searchResponse.communities[0].name).toBe('main');
let followForm: FollowCommunityForm = {
community_id: searchResponse.communities[0].id,
follow: true,
auth: lemmyAlphaAuth,
};
let followResAgain: CommunityResponse = await fetch(
`${lemmyAlphaApiUrl}/community/follow`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: wrapper(followForm),
}
).then(d => d.json());
// Make sure the follow response went through
expect(followResAgain.community.local).toBe(false);
expect(followResAgain.community.name).toBe('main');
let updatedCommentContent = 'An update child comment from beta';
let updatedCommentForm: CommentForm = {
content: updatedCommentContent,
post_id: createPostRes.post.id,
edit_id: createChildCommentRes.comment.id,
auth: lemmyBetaAuth,
creator_id: 2,
};
let updateResponse: CommentResponse = await fetch(
`${lemmyBetaApiUrl}/comment`,
{
method: 'PUT',
headers: {
'Content-Type': 'application/json',
},
body: wrapper(updatedCommentForm),
}
).then(d => d.json());
expect(updateResponse.comment.content).toBe(updatedCommentContent);
// Make sure that A picked up the post, parent comment, and child comment
let getPostUrl = `${lemmyAlphaApiUrl}/post?id=6`;
let getPostRes: GetPostResponse = await fetch(getPostUrl, {
method: 'GET',
}).then(d => d.json());
expect(getPostRes.post.name).toBe(betaPostName);
expect(getPostRes.comments[1].content).toBe(parentCommentContent);
expect(getPostRes.comments[0].content).toBe(updatedCommentContent);
expect(getPostRes.post.community_local).toBe(false);
expect(getPostRes.post.creator_local).toBe(false);
});
});
});
function wrapper(form: any): string {