LemmyNet/lemmy
2
0
Fork 0
mirror of https://github.com/LemmyNet/lemmy.git synced 2024-12-25 12:21:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Ignore errors when reading community outbox (fixes #1582)

Browse source
This commit is contained in:
Felix Ableitner 2021-10-20 13:50:44 +02:00
parent c2fa96a4ec
commit 795ad7a65d
7 changed files with 91 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
crates/apub/assets/lemmy-community.json
View file

@ -2,45 +2,49 @@
"@context": [ "@context": [
"https://www.w3.org/ns/activitystreams", "https://www.w3.org/ns/activitystreams",
{ {
"moderators": "as:moderators",
"sc": "http://schema.org#", "sc": "http://schema.org#",
"sensitive": "as:sensitive", "matrixUserId": {
"type": "sc:Text",
"id": "as:alsoKnownAs"
},
"stickied": "as:stickied",
"pt": "https://join-lemmy.org#", "pt": "https://join-lemmy.org#",
"comments_enabled": { "comments_enabled": {
"type": "sc:Boolean", "type": "sc:Boolean",
"id": "pt:commentsEnabled" "id": "pt:commentsEnabled"
}, },
"stickied": "as:stickied", "sensitive": "as:sensitive"
"moderators": "as:moderators",
"matrixUserId": {
"type": "sc:Text",
"id": "as:alsoKnownAs"
}
}, },
"https://w3id.org/security/v1" "https://w3id.org/security/v1"
], ],
"type": "Group", "type": "Group",
"id": "https://lemmy.ml/c/meta", "id": "https://lemmy.ml/c/announcements",
"preferredUsername": "meta", "preferredUsername": "announcements",
"name": "lemmy.ml meta", "name": "Announcements",
"content": "<p>Anything about the lemmy.ml instance and its moderation.</p>\n<p>For discussion about the Lemmy software project, go to <a href=\"https://lemmy.ml/c/lemmy\">!lemmy@lemmy.ml</a>.</p>\n", "content": "<p>Lemmy Announcements</p>\n<hr />\n<p>Feel free to announce new communities here.</p>\n<p>Other than that, this is <em>reserved for admin use only</em>.</p>\n",
"mediaType": "text/html", "mediaType": "text/html",
"source": { "source": {
"content": "Anything about the lemmy.ml instance and its moderation. \n\nFor discussion about the Lemmy software project, go to [!lemmy@lemmy.ml](https://lemmy.ml/c/lemmy).", "content": "Lemmy Announcements\n\n---\n\nFeel free to announce new communities here.\n\nOther than that, this is *reserved for admin use only*.",
"mediaType": "text/markdown" "mediaType": "text/markdown"
}, },
"icon": {
"type": "Image",
"url": "https://lemmy.ml/pictrs/image/waqyZwLAy4.webp"
},
"sensitive": false, "sensitive": false,
"moderators": "https://lemmy.ml/c/meta/moderators", "moderators": "https://lemmy.ml/c/announcements/moderators",
"inbox": "https://lemmy.ml/c/meta/inbox", "inbox": "https://lemmy.ml/c/announcements/inbox",
"outbox": "https://lemmy.ml/c/meta/outbox", "outbox": "https://lemmy.ml/c/announcements/outbox",
"followers": "https://lemmy.ml/c/meta/followers", "followers": "https://lemmy.ml/c/announcements/followers",
"endpoints": { "endpoints": {
"sharedInbox": "https://lemmy.ml/inbox" "sharedInbox": "https://lemmy.ml/inbox"
}, },
"publicKey": { "publicKey": {
"id": "https://lemmy.ml/c/meta#main-key", "id": "https://lemmy.ml/c/announcements#main-key",
"owner": "https://lemmy.ml/c/meta", "owner": "https://lemmy.ml/c/announcements",
"publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApLImCO5LEXMWb6lPCfCI\necpxU17j8CFfrDJLUWcuCmKO9KxJFmsv9uY2sYr9YX9CD4o+jnJ4D/e+/D+UeJiD\naJ3aYLEjLm+u6KuAGVREjSLzp1Js/6F2k0MIIQueiO5BBiootEY6pv3CwqqMC9L4\npWvIjmd03dMBhU1DTAblrj61oGj3c5l0OPdjeW7XnTPSrCZvaQ/1PWwVyUj0y/SC\nHiUiUiqUD88aPB8c7s85Svzs5RFT8WHcelPXJ7ydhYlU8R2tE1Octjy9Gb1R38k4\ns62LiwlRUAIzvnpMWOBi60aRpL8+IuZXoB2cCEyX5SQ+ETVp5DobHDgI0zzBEjfD\nkQIDAQAB\n-----END PUBLIC KEY-----\n" "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzRjKTNtvDCmugplwEh+g\nx1bhKm6BHUZfXfpscgMMm7tXFswSDzUQirMgfkxa9ubfr1PDFKffA2vQ9x6CyuO/\n70xTafdOHyV1tSqzgKz0ZvFZ/VCOo6qy1mYWVkrtBm/fKzM+87MdkKYB/zI4VyEJ\nLfLQgjwxBAEYUH3CBG71U0gO0TwbimWNN0vqlfp0QfThNe1WYObF88ZVzMLgFbr7\nRHBItZjlZ/d8foPDidlIR3l2dJjy0EsD8F9JM340jtX7LXqFmU4j1AQKNHTDLnUF\nwYVhzuQGNJ504l5LZkFG54XfIFT7dx2QwuuM9bSnfPv/98RYrq1Si6tCkxEt1cVe\n4wIDAQAB\n-----END PUBLIC KEY-----\n"
}, },
"published": "2021-06-14T22:50:50.985762+00:00", "published": "2019-06-02T16:43:50.799554+00:00",
"updated": "2021-06-14T23:29:55.801468+00:00" "updated": "2021-03-10T17:18:10.498868+00:00"
} }

16
crates/apub/assets/lemmy-person.json
View file

@ -2,19 +2,19 @@
"@context": [ "@context": [
"https://www.w3.org/ns/activitystreams", "https://www.w3.org/ns/activitystreams",
{ {
"matrixUserId": {
"type": "sc:Text",
"id": "as:alsoKnownAs"
},
"sc": "http://schema.org#",
"sensitive": "as:sensitive",
"stickied": "as:stickied", "stickied": "as:stickied",
"pt": "https://join-lemmy.org#",
"comments_enabled": { "comments_enabled": {
"type": "sc:Boolean", "type": "sc:Boolean",
"id": "pt:commentsEnabled" "id": "pt:commentsEnabled"
}, },
"pt": "https://join-lemmy.org#", "moderators": "as:moderators"
"sensitive": "as:sensitive",
"moderators": "as:moderators",
"sc": "http://schema.org#",
"matrixUserId": {
"type": "sc:Text",
"id": "as:alsoKnownAs"
}
}, },
"https://w3id.org/security/v1" "https://w3id.org/security/v1"
], ],

26
crates/apub/assets/lemmy-post.json
View file

@ -2,40 +2,38 @@
"@context": [ "@context": [
"https://www.w3.org/ns/activitystreams", "https://www.w3.org/ns/activitystreams",
{ {
"moderators": "as:moderators",
"matrixUserId": { "matrixUserId": {
"type": "sc:Text", "type": "sc:Text",
"id": "as:alsoKnownAs" "id": "as:alsoKnownAs"
}, },
"sensitive": "as:sensitive",
"sc": "http://schema.org#", "sc": "http://schema.org#",
"pt": "https://join-lemmy.org#",
"comments_enabled": { "comments_enabled": {
"type": "sc:Boolean", "type": "sc:Boolean",
"id": "pt:commentsEnabled" "id": "pt:commentsEnabled"
}, },
"sensitive": "as:sensitive", "stickied": "as:stickied"
"stickied": "as:stickied",
"pt": "https://join-lemmy.org#",
"moderators": "as:moderators"
}, },
"https://w3id.org/security/v1" "https://w3id.org/security/v1"
], ],
"type": "Page", "type": "Page",
"id": "https://lemmy.ml/post/70280", "id": "https://lemmy.ml/post/55143",
"attributedTo": "attributedTo": "https://lemmy.ml/u/nutomic",
"https://lemmy.ml/u/nutomic"
,
"to": [ "to": [
"https://lemmy.ml/c/meta", "https://lemmy.ml/c/announcements",
"https://www.w3.org/ns/activitystreams#Public" "https://www.w3.org/ns/activitystreams#Public"
], ],
"name": "What is lemmy.ml?", "name": "Statement on Politics of Lemmy.ml",
"content": "<p>Recently there seems to be some of misunderstanding what the lemmy.ml instance is about, especially from newer users.</p>\n<p>Lemmy.ml has always been a niche site, and it will most likely stay this way. We don't have any intentions to turn it into a mainstream instance, or set a goal of getting as many users as possible. Our goal is simple: make an instance that people like to use. I would say that we have been successful in this, but obviously it is impossible to satisfy everyone.</p>\n<p>The reason for this is that @dessalines and I are paid to develop Lemmy, while donations from lemmy.ml users only make up a negligible part of our income. Besides, having more users would force us to spend more time moderating, and less time for development. Lemmy works quite differently from big tech sites like Reddit in this regard: while they get more money with each extra user through advertising, for us it is the opposite. So we would much rather have a smaller, non-toxic, and friendly userbase, than a large one.</p>\n<p>Part of the problem might be that lemmy.ml is described as &quot;flagship instance&quot;, which can certainly be interpreted to mean &quot;mainstream&quot; or &quot;general purpose&quot;. I struggle to come up with a better, more accurate description. If you can think of one, please comment here.</p>\n<p>If you dont like the way lemmy.ml works, thats okay. Federation exists exactly to solve that problem, let different groups have their own instances, with their own rules and political views. You can see the list of existing instances, and instructions for setting up a new one on <a href=\"https://join-lemmy.org/\">join-lemmy.org</a>.</p>\n<p>In particular, I would like to see someone (or a group of people) create a mainstream, or liberal instance. That should help to avoid further drama, and avoid attempts to turn lemmy.ml into something that it is not. @dessalines and I would certainly be willing to help with any technical problems that such an instance runs into, and include it on <a href=\"https://join-lemmy.org/\">join-lemmy.org</a> (just like any other instance that meets the code of conduct).</p>\n", "content": "<p>Recently there have been some discussions about the political stances of the Lemmy developers and site admins. To clear up some misconceptions: Lemmy is run by a team of people with different ideologies, including anti-capitalist, communist, anarchist, and others. While @dessalines and I are communists, we take decisions collectively, and don't demand that anyone adopt our views or convert to our ideologies. We wouldn't devote so much time to building a federated site otherwise.</p>\n<p>What's important to us is that you follow the site rules and <a href=\"https://join.lemmy.ml/docs/en/code_of_conduct.html\">Code of Conduct</a>. Meaning primarily, no-bigotry, and being respectful towards others. As long as that is the case, we can get along perfectly fine.</p>\n<p>In general we are open for constructive feedback, so please contact any member of the admin team if you have an idea how to improve Lemmy.</p>\n<h2>Slur Filter</h2>\n<p>We also noticed a consistent criticism of the built-in slur filter in Lemmy. Not so much on lemmy.ml itself, but whenever Lemmy is recommended elsewhere, a few usual suspects keep bringing it up. To these people we say the following: we are using the slur filter as a tool to keep a friendly atmosphere, and prevent racists, sexists and other bigots from using Lemmy. Its existence alone has lead many of them to not make an account, or run an instance: a clear net positive.</p>\n<p>You can see for yourself the words which are blocked (content warning, <a href=\"https://github.com/LemmyNet/lemmy/blob/main/crates/utils/src/utils.rs#L10\">link here</a>). Note that it doesn't include any simple swear words, but only slurs which are used to insult and attack other people. If you want to use any of these words, then please stay on one of the many platforms that permit them. Lemmy is not for you, and we don't want you here.</p>\n<p>We are fully aware that the slur filter is not perfect. It is made for American English, and can give false positives in other languages or dialects. We are totally willing to fix such problems on a case by case basis, simply open an issue in our <a href=\"https://github.com/LemmyNet/lemmy/issues\">repo </a>with a description of the problem.</p>\n",
"mediaType": "text/html", "mediaType": "text/html",
"source": { "source": {
"content": "Recently there seems to be some of misunderstanding what the lemmy.ml instance is about, especially from newer users.\n\nLemmy.ml has always been a niche site, and it will most likely stay this way. We don't have any intentions to turn it into a mainstream instance, or set a goal of getting as many users as possible. Our goal is simple: make an instance that people like to use. I would say that we have been successful in this, but obviously it is impossible to satisfy everyone.\n\nThe reason for this is that @dessalines and I are paid to develop Lemmy, while donations from lemmy.ml users only make up a negligible part of our income. Besides, having more users would force us to spend more time moderating, and less time for development. Lemmy works quite differently from big tech sites like Reddit in this regard: while they get more money with each extra user through advertising, for us it is the opposite. So we would much rather have a smaller, non-toxic, and friendly userbase, than a large one.\n\nPart of the problem might be that lemmy.ml is described as \"flagship instance\", which can certainly be interpreted to mean \"mainstream\" or \"general purpose\". I struggle to come up with a better, more accurate description. If you can think of one, please comment here.\n\nIf you dont like the way lemmy.ml works, thats okay. Federation exists exactly to solve that problem, let different groups have their own instances, with their own rules and political views. You can see the list of existing instances, and instructions for setting up a new one on [join-lemmy.org](https://join-lemmy.org/).\n\nIn particular, I would like to see someone (or a group of people) create a mainstream, or liberal instance. That should help to avoid further drama, and avoid attempts to turn lemmy.ml into something that it is not. @dessalines and I would certainly be willing to help with any technical problems that such an instance runs into, and include it on [join-lemmy.org](https://join-lemmy.org/) (just like any other instance that meets the code of conduct).", "content": "Recently there have been some discussions about the political stances of the Lemmy developers and site admins. To clear up some misconceptions: Lemmy is run by a team of people with different ideologies, including anti-capitalist, communist, anarchist, and others. While @dessalines and I are communists, we take decisions collectively, and don't demand that anyone adopt our views or convert to our ideologies. We wouldn't devote so much time to building a federated site otherwise.\n\nWhat's important to us is that you follow the site rules and [Code of Conduct](https://join.lemmy.ml/docs/en/code_of_conduct.html). Meaning primarily, no-bigotry, and being respectful towards others. As long as that is the case, we can get along perfectly fine.\n\nIn general we are open for constructive feedback, so please contact any member of the admin team if you have an idea how to improve Lemmy. \n\n## Slur Filter\n\nWe also noticed a consistent criticism of the built-in slur filter in Lemmy. Not so much on lemmy.ml itself, but whenever Lemmy is recommended elsewhere, a few usual suspects keep bringing it up. To these people we say the following: we are using the slur filter as a tool to keep a friendly atmosphere, and prevent racists, sexists and other bigots from using Lemmy. Its existence alone has lead many of them to not make an account, or run an instance: a clear net positive.\n\nYou can see for yourself the words which are blocked (content warning, [link here](https://github.com/LemmyNet/lemmy/blob/main/crates/utils/src/utils.rs#L10)). Note that it doesn't include any simple swear words, but only slurs which are used to insult and attack other people. If you want to use any of these words, then please stay on one of the many platforms that permit them. Lemmy is not for you, and we don't want you here.\n\nWe are fully aware that the slur filter is not perfect. It is made for American English, and can give false positives in other languages or dialects. We are totally willing to fix such problems on a case by case basis, simply open an issue in our [repo ](https://github.com/LemmyNet/lemmy/issues)with a description of the problem.",
"mediaType": "text/markdown" "mediaType": "text/markdown"
}, },
"commentsEnabled": false, "commentsEnabled": true,
"sensitive": false, "sensitive": false,
"stickied": true, "stickied": true,
"published": "2021-06-15T12:00:08.364559+00:00" "published": "2021-02-26T12:35:34.292626+00:00"
} }

27
crates/apub/src/fetcher/community.rs
View file

@ -3,7 +3,10 @@ use crate::{
fetcher::{fetch::fetch_remote_object, object_id::ObjectId}, fetcher::{fetch::fetch_remote_object, object_id::ObjectId},
objects::{community::Group, person::ApubPerson}, objects::{community::Group, person::ApubPerson},
}; };
use activitystreams::collection::{CollectionExt, OrderedCollection}; use activitystreams::{
base::AnyBase,
collection::{CollectionExt, OrderedCollection},
};
use anyhow::Context; use anyhow::Context;
use lemmy_api_common::blocking; use lemmy_api_common::blocking;
use lemmy_apub_lib::{data::Data, traits::ActivityHandler}; use lemmy_apub_lib::{data::Data, traits::ActivityHandler};
@ -85,16 +88,30 @@ pub(crate) async fn fetch_community_outbox(
outbox_activities = outbox_activities[0..20].to_vec(); outbox_activities = outbox_activities[0..20].to_vec();
} }
for announce in outbox_activities { // We intentionally ignore errors here. This is because the outbox might contain posts from old
// Lemmy versions, or from other software which we cant parse. In that case, we simply skip the
// item and only parse the ones that work.
for activity in outbox_activities {
parse_outbox_item(activity, context, recursion_counter)
.await
.ok();
}
Ok(())
}
async fn parse_outbox_item(
announce: AnyBase,
context: &LemmyContext,
request_counter: &mut i32,
) -> Result<(), LemmyError> {
// TODO: instead of converting like this, we should create a struct CommunityOutbox with // TODO: instead of converting like this, we should create a struct CommunityOutbox with
// AnnounceActivity as inner type, but that gives me stackoverflow // AnnounceActivity as inner type, but that gives me stackoverflow
let ser = serde_json::to_string(&announce)?; let ser = serde_json::to_string(&announce)?;
let announce: AnnounceActivity = serde_json::from_str(&ser)?; let announce: AnnounceActivity = serde_json::from_str(&ser)?;
announce announce
.receive(&Data::new(context.clone()), recursion_counter) .receive(&Data::new(context.clone()), request_counter)
.await?; .await?;
}
Ok(()) Ok(())
} }

6
crates/apub/src/objects/community.rs
View file

@ -315,17 +315,17 @@ mod tests {
#[serial] #[serial]
async fn test_fetch_lemmy_community() { async fn test_fetch_lemmy_community() {
let json = file_to_json_object("assets/lemmy-community.json"); let json = file_to_json_object("assets/lemmy-community.json");
let url = Url::parse("https://lemmy.ml/c/meta").unwrap(); let url = Url::parse("https://lemmy.ml/c/announcements").unwrap();
let mut request_counter = 0; let mut request_counter = 0;
let community = ApubCommunity::from_apub(&json, &init_context(), &url, &mut request_counter) let community = ApubCommunity::from_apub(&json, &init_context(), &url, &mut request_counter)
.await .await
.unwrap(); .unwrap();
assert_eq!(community.actor_id.clone().into_inner(), url); assert_eq!(community.actor_id.clone().into_inner(), url);
assert_eq!(community.title, "lemmy.ml meta"); assert_eq!(community.title, "Announcements");
assert!(community.public_key.is_some()); assert!(community.public_key.is_some());
assert!(!community.local); assert!(!community.local);
assert_eq!(community.description.as_ref().unwrap().len(), 158); assert_eq!(community.description.as_ref().unwrap().len(), 126);
// TODO: its fetching the outbox, mod collection, and probably users from the outbox. due to // TODO: its fetching the outbox, mod collection, and probably users from the outbox. due to
// caching and other things, this may change over multiple runs, so we cant assert it. // caching and other things, this may change over multiple runs, so we cant assert it.
// find a way to avoid any network requests // find a way to avoid any network requests

22
crates/apub/src/objects/post.rs
View file

@ -280,24 +280,32 @@ impl FromApub for ApubPost {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::objects::tests::{file_to_json_object, init_context}; use crate::objects::{
community::ApubCommunity,
tests::{file_to_json_object, init_context},
};
use serial_test::serial; use serial_test::serial;
#[actix_rt::test] #[actix_rt::test]
#[serial] #[serial]
async fn test_fetch_lemmy_post() { async fn test_fetch_lemmy_post() {
let context = init_context();
let url = Url::parse("https://lemmy.ml/post/55143").unwrap();
let mut request_counter = -20;
let community_json = file_to_json_object("assets/lemmy-community.json");
ApubCommunity::from_apub(&community_json, &context, &url, &mut request_counter)
.await
.unwrap();
let json = file_to_json_object("assets/lemmy-post.json"); let json = file_to_json_object("assets/lemmy-post.json");
let url = Url::parse("https://lemmy.ml/post/70280").unwrap(); let post = ApubPost::from_apub(&json, &context, &url, &mut request_counter)
let mut request_counter = 0;
let post = ApubPost::from_apub(&json, &init_context(), &url, &mut request_counter)
.await .await
.unwrap(); .unwrap();
assert_eq!(post.ap_id.clone().into_inner(), url); assert_eq!(post.ap_id.clone().into_inner(), url);
assert_eq!(post.name, "What is lemmy.ml?"); assert_eq!(post.name, "Statement on Politics of Lemmy.ml");
assert!(post.body.is_some()); assert!(post.body.is_some());
assert_eq!(post.body.as_ref().unwrap().len(), 2052); assert_eq!(post.body.as_ref().unwrap().len(), 2144);
assert!(post.locked); assert!(!post.locked);
assert!(post.stickied); assert!(post.stickied);
// see comment in test_fetch_lemmy_community() about this // see comment in test_fetch_lemmy_community() about this
//assert_eq!(request_counter, 5); //assert_eq!(request_counter, 5);

4
scripts/update_apub_test_files.sh
View file

@ -3,9 +3,9 @@ set -e
curl -H "Accept: application/activity+json" https://lemmy.ml/u/nutomic | jq \ curl -H "Accept: application/activity+json" https://lemmy.ml/u/nutomic | jq \
> crates/apub/assets/lemmy-person.json > crates/apub/assets/lemmy-person.json
curl -H "Accept: application/activity+json" https://lemmy.ml/c/meta | jq \ curl -H "Accept: application/activity+json" https://lemmy.ml/c/announcements | jq \
> crates/apub/assets/lemmy-community.json > crates/apub/assets/lemmy-community.json
curl -H "Accept: application/activity+json" https://lemmy.ml/post/70280 | jq \ curl -H "Accept: application/activity+json" https://lemmy.ml/post/55143 | jq \
> crates/apub/assets/lemmy-post.json > crates/apub/assets/lemmy-post.json
curl -H "Accept: application/activity+json" https://queer.hacktivis.me/users/lanodan | jq \ curl -H "Accept: application/activity+json" https://queer.hacktivis.me/users/lanodan | jq \
> crates/apub/assets/pleroma-person.json > crates/apub/assets/pleroma-person.json