Allow fetching person from Pleroma, including test case (ref #1461)

Cargo.lock

@@ -1785,6 +1785,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_with",
+ "serial_test",
  "sha2",
  "strum",
  "strum_macros",

config/config.hjson

@@ -1,5 +1,8 @@
 # See the documentation for available config fields and descriptions:
 # https://join-lemmy.org/docs/en/administration/configuration.html
 {
-  hostname: lemmy-alpha
+  hostname: lemmy-alphan
+  federation: {
+    enabled: true
+  }
 }

crates/apub/Cargo.toml

@@ -50,3 +50,5 @@ thiserror = "1.0.29"
 background-jobs = "0.9.0"
 reqwest = { version = "0.11.4", features = ["json"] }
 
+[dev-dependencies]
+serial_test = "0.5.1"

crates/apub/assets/lemmy-person.json

@@ -0,0 +1,46 @@
+{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "moderators": "as:moderators",
+      "sensitive": "as:sensitive",
+      "pt": "https://join-lemmy.org#",
+      "sc": "http://schema.org#",
+      "stickied": "as:stickied",
+      "matrixUserId": {
+        "type": "sc:Text",
+        "id": "as:alsoKnownAs"
+      },
+      "comments_enabled": {
+        "type": "sc:Boolean",
+        "id": "pt:commentsEnabled"
+      }
+    },
+    "https://w3id.org/security/v1"
+  ],
+  "type": "Person",
+  "id": "https://lemmy.ml/u/nutomic",
+  "preferredUsername": "nutomic",
+  "content": "<p>Lemmy maintainer. Interested in politics, video games, and many other things.</p>\n",
+  "mediaType": "text/html",
+  "source": {
+    "content": "Lemmy maintainer. Interested in politics, video games, and many other things.",
+    "mediaType": "text/markdown"
+  },
+  "icon": {
+    "type": "Image",
+    "url": "https://lemmy.ml/pictrs/image/ed9ej7.jpg"
+  },
+  "inbox": "https://lemmy.ml/u/nutomic/inbox",
+  "outbox": "https://lemmy.ml/u/nutomic/outbox",
+  "endpoints": {
+    "sharedInbox": "https://lemmy.ml/inbox"
+  },
+  "publicKey": {
+    "id": "https://lemmy.ml/u/nutomic#main-key",
+    "owner": "https://lemmy.ml/u/nutomic",
+    "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0lP99/s5Vv+XbPdkeqIJ\nwoD4GFnHmBnBHdEKChEUWfWj1TtioC/rGNoXFQeXQA3Amhy4nxSceiDnUgwkkuQY\nv0MtIW58NzgknEavtllxL+LSds5pg3gANaDIk8UiWTkqXTg0GnlJMpCK1Chen0l/\nszL6DEvUyTSuS5ZYDXFgewF89Pe7U0S15V5U2Harv7AgJYDyxmUL0D1pGuUCRqcE\nl5MTHJjrXeNnH1w2g8aly8YlO/Cr0L51rFg/lBF23vni7ZLv8HbmWh6YpaAf1R8h\nE45zKR7OHqymdjzrg1ITBwovefpwMkVgnJ+Wdr4HPnFlBSkXPoZeM11+Z8L0anzA\nXwIDAQAB\n-----END PUBLIC KEY-----\n"
+  },
+  "published": "2020-01-17T01:38:22.348392+00:00",
+  "updated": "2021-08-13T00:11:15.941990+00:00"
+}

crates/apub/assets/pleroma-person.json

@@ -0,0 +1,79 @@
+{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    "https://queer.hacktivis.me/schemas/litepub-0.1.jsonld",
+    {
+      "@language": "und"
+    }
+  ],
+  "alsoKnownAs": [],
+  "attachment": [],
+  "capabilities": {
+    "acceptsChatMessages": true
+  },
+  "discoverable": false,
+  "endpoints": {
+    "oauthAuthorizationEndpoint": "https://queer.hacktivis.me/oauth/authorize",
+    "oauthRegistrationEndpoint": "https://queer.hacktivis.me/api/v1/apps",
+    "oauthTokenEndpoint": "https://queer.hacktivis.me/oauth/token",
+    "sharedInbox": "https://queer.hacktivis.me/inbox",
+    "uploadMedia": "https://queer.hacktivis.me/api/ap/upload_media"
+  },
+  "featured": "https://queer.hacktivis.me/users/lanodan/collections/featured",
+  "followers": "https://queer.hacktivis.me/users/lanodan/followers",
+  "following": "https://queer.hacktivis.me/users/lanodan/following",
+  "icon": {
+    "type": "Image",
+    "url": "https://queer.hacktivis.me/media/d23cf9b0-5586-4592-aca5-9a52777a6042/avatar_HD.png"
+  },
+  "id": "https://queer.hacktivis.me/users/lanodan",
+  "image": {
+    "type": "Image",
+    "url": "https://queer.hacktivis.me/media/37b6ce56-8c24-4e64-bd70-a76e84ab0c69/53a48a3a49ed5e5637a84e4f3663df17f8d764244bbc1027ba03cfc446e8b7bd.jpg"
+  },
+  "inbox": "https://queer.hacktivis.me/users/lanodan/inbox",
+  "manuallyApprovesFollowers": false,
+  "name": "Haelwenn /элвэн/ :bzh: ",
+  "outbox": "https://queer.hacktivis.me/users/lanodan/outbox",
+  "preferredUsername": "lanodan",
+  "publicKey": {
+    "id": "https://queer.hacktivis.me/users/lanodan#main-key",
+    "owner": "https://queer.hacktivis.me/users/lanodan",
+    "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsWOgdjSMc010qvxC3njI\nXJlFWMJ5gJ8QXCW/PajYdsHPM6d+jxBNJ6zp9/tIRa2m7bWHTSkuHQ7QthOpt6vu\n+dAWpKRLS607SPLItn/qUcyXvgN+H8shfyhMxvkVs9jXdtlBsLUVE7UNpN0dxzqe\nI79QWbf7o4amgaIWGRYB+OYMnIxKt+GzIkivZdSVSYjfxNnBYkMCeUxm5EpPIxKS\nP5bBHAVRRambD5NUmyKILuC60/rYuc/C+vmgpY2HCWFS2q6o34dPr9enwL6t4b3m\nS1t/EJHk9rGaaDqSGkDEfyQI83/7SDebWKuETMKKFLZi1vMgQIFuOYCIhN6bIiZm\npQIDAQAB\n-----END PUBLIC KEY-----\n\n"
+  },
+  "summary": "---<br/>Website: <a href=\"https://hacktivis.me/\">https://hacktivis.me/</a><br/>Lang: Français(natif), English(fluent), LSF(🤏~👌), русский (еле-еле), <br/>Politics: Anarchist as in DIY/DIWO, freedom of association, anti-authoritarian, anti-identitarianism<br/><br/>Pronouns: meh, pick any, have fun<br/>Timezone: Let&#39;s say Mars, I have a non-24h cycle<br/>```<br/>🦊🦄⚧🂡ⓥ :anarchy: 👿🐧 :gentoo:<br/>Pleroma maintainer (mostly backend)<br/>BadWolf developer<br/>Gentoo contributor<br/><br/>Dayjob: yogoko.fr<br/><br/>That person which uses HJKL in games<br/><br/>Just because computer bad: X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*<br/><br/>banner from: <a href=\"https://soc.flyingcube.tech/objects/56f79be2-9013-4559-9826-f7dc392417db\">https://soc.flyingcube.tech/objects/56f79be2-9013-4559-9826-f7dc392417db</a><br/>Federation-bots: <a class=\"hashtag\" data-tag=\"nobot\" href=\"https://queer.hacktivis.me/tag/nobot\">#nobot</a>",
+  "tag": [
+    {
+      "icon": {
+        "type": "Image",
+        "url": "https://queer.hacktivis.me/emoji/custom/symbols/anarchy.png"
+      },
+      "id": "https://queer.hacktivis.me/emoji/custom/symbols/anarchy.png",
+      "name": ":anarchy:",
+      "type": "Emoji",
+      "updated": "1970-01-01T00:00:00Z"
+    },
+    {
+      "icon": {
+        "type": "Image",
+        "url": "https://queer.hacktivis.me/emoji/custom/mastodon.xyz/bzh.png"
+      },
+      "id": "https://queer.hacktivis.me/emoji/custom/mastodon.xyz/bzh.png",
+      "name": ":bzh:",
+      "type": "Emoji",
+      "updated": "1970-01-01T00:00:00Z"
+    },
+    {
+      "icon": {
+        "type": "Image",
+        "url": "https://queer.hacktivis.me/emoji/custom/gentoo.png"
+      },
+      "id": "https://queer.hacktivis.me/emoji/custom/gentoo.png",
+      "name": ":gentoo:",
+      "type": "Emoji",
+      "updated": "1970-01-01T00:00:00Z"
+    }
+  ],
+  "type": "Person",
+  "url": "https://queer.hacktivis.me/users/lanodan"
+}

crates/apub/src/objects/comment.rs

@@ -60,7 +60,7 @@ pub struct Note {
   media_type: MediaTypeHtml,
   source: Source,
   in_reply_to: CommentInReplyToMigration,
-  published: DateTime<FixedOffset>,
+  published: Option<DateTime<FixedOffset>>,
   updated: Option<DateTime<FixedOffset>>,
   #[serde(flatten)]
   unparsed: Unparsed,
@@ -230,7 +230,7 @@ impl ToApub for ApubComment {
         media_type: MediaTypeMarkdown::Markdown,
       },
       in_reply_to: CommentInReplyToMigration::Old(in_reply_to_vec),
-      published: convert_datetime(self.published),
+      published: Some(convert_datetime(self.published)),
       updated: self.updated.map(convert_datetime),
       unparsed: Default::default(),
     };
@@ -282,7 +282,7 @@ impl FromApub for ApubComment {
       content: content_slurs_removed,
       removed: None,
       read: None,
-      published: Some(note.published.naive_local()),
+      published: note.published.map(|u| u.to_owned().naive_local()),
       updated: note.updated.map(|u| u.to_owned().naive_local()),
       deleted: None,
       ap_id,

crates/apub/src/objects/community.rs

@@ -69,7 +69,7 @@ pub struct Group {
   followers: Url,
   endpoints: Endpoints<Url>,
   public_key: PublicKey,
-  published: DateTime<FixedOffset>,
+  published: Option<DateTime<FixedOffset>>,
   updated: Option<DateTime<FixedOffset>>,
   #[serde(flatten)]
   unparsed: Unparsed,
@@ -101,7 +101,7 @@ impl Group {
       title,
       description,
       removed: None,
-      published: Some(group.published.naive_local()),
+      published: group.published.map(|u| u.naive_local()),
       updated: group.updated.map(|u| u.naive_local()),
       deleted: None,
       nsfw: Some(group.sensitive.unwrap_or(false)),
@@ -232,7 +232,7 @@ impl ToApub for ApubCommunity {
         ..Default::default()
       },
       public_key: self.get_public_key()?,
-      published: convert_datetime(self.published),
+      published: Some(convert_datetime(self.published)),
       updated: self.updated.map(convert_datetime),
       unparsed: Default::default(),
     };

crates/apub/src/objects/mod.rs

@@ -53,3 +53,80 @@ where
     Err(anyhow!("Cant convert object to tombstone if it wasnt deleted").into())
   }
 }
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+  use actix::Actor;
+  use diesel::{
+    r2d2::{ConnectionManager, Pool},
+    PgConnection,
+  };
+  use lemmy_apub_lib::activity_queue::create_activity_queue;
+  use lemmy_db_schema::{
+    establish_unpooled_connection,
+    get_database_url_from_env,
+    source::secret::Secret,
+  };
+  use lemmy_utils::{
+    rate_limit::{rate_limiter::RateLimiter, RateLimit},
+    request::build_user_agent,
+    settings::structs::Settings,
+  };
+  use lemmy_websocket::{chat_server::ChatServer, LemmyContext};
+  use reqwest::Client;
+  use serde::de::DeserializeOwned;
+  use std::{fs::File, io::BufReader, sync::Arc};
+  use tokio::sync::Mutex;
+
+  // TODO: would be nice if we didnt have to use a full context for tests.
+  //       or at least write a helper function so this code is shared with main.rs
+  pub(crate) fn init_context() -> LemmyContext {
+    // call this to run migrations
+    establish_unpooled_connection();
+    let settings = Settings::init().unwrap();
+    let rate_limiter = RateLimit {
+      rate_limiter: Arc::new(Mutex::new(RateLimiter::default())),
+      rate_limit_config: settings.rate_limit.to_owned().unwrap_or_default(),
+    };
+    let client = Client::builder()
+      .user_agent(build_user_agent(&settings))
+      .build()
+      .unwrap();
+    let activity_queue = create_activity_queue();
+    let secret = Secret {
+      id: 0,
+      jwt_secret: "".to_string(),
+    };
+    let db_url = match get_database_url_from_env() {
+      Ok(url) => url,
+      Err(_) => settings.get_database_url(),
+    };
+    let manager = ConnectionManager::<PgConnection>::new(&db_url);
+    let pool = Pool::builder()
+      .max_size(settings.database.pool_size)
+      .build(manager)
+      .unwrap_or_else(|_| panic!("Error connecting to {}", db_url));
+    async fn x() -> Result<String, LemmyError> {
+      Ok("".to_string())
+    }
+    let chat_server = ChatServer::startup(
+      pool.clone(),
+      rate_limiter.clone(),
+      |_, _, _, _| Box::pin(x()),
+      |_, _, _, _| Box::pin(x()),
+      client.clone(),
+      activity_queue.clone(),
+      settings.clone(),
+      secret.clone(),
+    )
+    .start();
+    LemmyContext::create(pool, chat_server, client, activity_queue, settings, secret)
+  }
+
+  pub(crate) fn file_to_json_object<T: DeserializeOwned>(path: &str) -> T {
+    let file = File::open(path).unwrap();
+    let reader = BufReader::new(file);
+    serde_json::from_reader(reader).unwrap()
+  }
+}

crates/apub/src/objects/person.rs

@@ -67,7 +67,7 @@ pub struct Person {
   outbox: Url,
   endpoints: Endpoints<Url>,
   public_key: PublicKey,
-  published: DateTime<FixedOffset>,
+  published: Option<DateTime<FixedOffset>>,
   updated: Option<DateTime<FixedOffset>>,
   #[serde(flatten)]
   unparsed: Unparsed,
@@ -192,7 +192,7 @@ impl ToApub for ApubPerson {
       icon,
       image,
       matrix_user_id: self.matrix_user_id.clone(),
-      published: convert_datetime(self.published),
+      published: Some(convert_datetime(self.published)),
       outbox: generate_outbox_url(&self.actor_id)?.into(),
       endpoints: Endpoints {
         shared_inbox: self.shared_inbox_url.clone().map(|s| s.into()),
@@ -245,7 +245,7 @@ impl FromApub for ApubPerson {
       deleted: None,
       avatar: Some(person.icon.clone().map(|i| i.url.into())),
       banner: Some(person.image.clone().map(|i| i.url.into())),
-      published: Some(person.published.naive_local()),
+      published: person.published.map(|u| u.clone().naive_local()),
       updated: person.updated.map(|u| u.clone().naive_local()),
       actor_id,
       bio: Some(bio),
@@ -266,3 +266,43 @@ impl FromApub for ApubPerson {
     Ok(person.into())
   }
 }
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+  use crate::objects::tests::{file_to_json_object, init_context};
+  use serial_test::serial;
+
+  #[actix_rt::test]
+  #[serial]
+  async fn test_fetch_lemmy_person() {
+    let json = file_to_json_object("assets/lemmy-person.json");
+    let url = Url::parse("https://lemmy.ml/u/nutomic").unwrap();
+    let person = ApubPerson::from_apub(&json, &init_context(), &url, &mut 0)
+      .await
+      .unwrap();
+
+    assert_eq!(person.actor_id.clone().into_inner(), url);
+    assert_eq!(person.name, "nutomic");
+    assert!(person.public_key.is_some());
+    assert_eq!(person.local, false);
+    assert!(person.bio.is_some());
+  }
+
+  #[actix_rt::test]
+  #[serial]
+  async fn test_fetch_pleroma_person() {
+    let json = file_to_json_object("assets/pleroma-person.json");
+    let url = Url::parse("https://queer.hacktivis.me/users/lanodan").unwrap();
+    let person = ApubPerson::from_apub(&json, &init_context(), &url, &mut 0)
+      .await
+      .unwrap();
+
+    assert_eq!(person.actor_id.clone().into_inner(), url);
+    assert_eq!(person.name, "lanodan");
+    assert!(person.public_key.is_some());
+    assert_eq!(person.local, false);
+    // TODO: pleroma uses summary for user profile, while we use content
+    //assert!(person.bio.is_some());
+  }
+}

crates/apub/src/objects/post.rs

@@ -61,7 +61,7 @@ pub struct Page {
   pub(crate) comments_enabled: Option<bool>,
   sensitive: Option<bool>,
   pub(crate) stickied: Option<bool>,
-  published: DateTime<FixedOffset>,
+  published: Option<DateTime<FixedOffset>>,
   updated: Option<DateTime<FixedOffset>>,
   #[serde(flatten)]
   unparsed: Unparsed,
@@ -196,7 +196,7 @@ impl ToApub for ApubPost {
       comments_enabled: Some(!self.locked),
       sensitive: Some(self.nsfw),
       stickied: Some(self.stickied),
-      published: convert_datetime(self.published),
+      published: Some(convert_datetime(self.published)),
       updated: self.updated.map(convert_datetime),
       unparsed: Default::default(),
     };
@@ -260,7 +260,7 @@ impl FromApub for ApubPost {
       community_id: community.id,
       removed: None,
       locked: page.comments_enabled.map(|e| !e),
-      published: Some(page.published.naive_local()),
+      published: page.published.map(|u| u.naive_local()),
       updated: page.updated.map(|u| u.naive_local()),
       deleted: None,
       nsfw: page.sensitive,

crates/apub/src/objects/private_message.rs

@@ -46,7 +46,7 @@ pub struct Note {
   content: String,
   media_type: MediaTypeHtml,
   source: Source,
-  published: DateTime<FixedOffset>,
+  published: Option<DateTime<FixedOffset>>,
   updated: Option<DateTime<FixedOffset>>,
   #[serde(flatten)]
   unparsed: Unparsed,
@@ -146,7 +146,7 @@ impl ToApub for ApubPrivateMessage {
         content: self.content.clone(),
         media_type: MediaTypeMarkdown::Markdown,
       },
-      published: convert_datetime(self.published),
+      published: Some(convert_datetime(self.published)),
       updated: self.updated.map(convert_datetime),
       unparsed: Default::default(),
     };
@@ -185,7 +185,7 @@ impl FromApub for ApubPrivateMessage {
       creator_id: creator.id,
       recipient_id: recipient.id,
       content: note.source.content.clone(),
-      published: Some(note.published.naive_local()),
+      published: note.published.map(|u| u.to_owned().naive_local()),
       updated: note.updated.map(|u| u.to_owned().naive_local()),
       deleted: None,
       read: None,

crates/websocket/src/lib.rs

@@ -16,12 +16,12 @@ pub mod routes;
 pub mod send;
 
 pub struct LemmyContext {
-  pub pool: DbPool,
+  pool: DbPool,
-  pub chat_server: Addr<ChatServer>,
+  chat_server: Addr<ChatServer>,
-  pub client: Client,
+  client: Client,
-  pub activity_queue: QueueHandle,
+  activity_queue: QueueHandle,
-  pub settings: Settings,
+  settings: Settings,
-  pub secret: Secret,
+  secret: Secret,
 }
 
 impl LemmyContext {

scripts/update_apub_test_files.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+
+curl -H "Accept: application/activity+json" https://lemmy.ml/u/nutomic | jq \
+    > crates/apub/assets/lemmy-person.json
+curl -H "Accept: application/activity+json" https://queer.hacktivis.me/users/lanodan | jq \
+    > crates/apub/assets/pleroma-person.json