Merge pull request 'Populate content with HTML, and source with markdown (ref #1220)' (#141) from apub-media-type2 into main

Reviewed-on: https://yerbamate.ml/LemmyNet/lemmy/pulls/141

.dockerignore

@@ -1,8 +1,8 @@
 # build folders and similar which are not needed for the docker build
 target
-docker/dev/volumes
+docker
-docker/prod/volumes
+api_tests
-docker/federation/volumes
-docker/travis/volumes
-.git
 ansible
+tests
+.git
+*.sh

Cargo.lock

@@ -2,9 +2,9 @@
 # It is not intended for manual editing.
 [[package]]
 name = "activitystreams"
-version = "0.7.0-alpha.4"
+version = "0.7.0-alpha.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "261b423734cca2a170d7a76936f1f0f9e6c6fc297d36cfc5ea6aa15f9017f996"
+checksum = "0b1afe32371e466a791ced0d6ef6e6b97822bb1a279ee4cc41c4324e61cd0b2b"
 dependencies = [
  "chrono",
  "mime",

docker/dev/Dockerfile

@@ -1,35 +1,72 @@
-# syntax=docker/dockerfile:experimental
+ARG RUST_BUILDER_IMAGE=ekidd/rust-musl-builder:1.47.0
-FROM rust:1.47-buster as rust
 
-ENV HOME=/home/root
+# Cargo chef plan
+FROM $RUST_BUILDER_IMAGE as planner
+WORKDIR /app
+RUN cargo install cargo-chef --version 0.1.6
+
+# Copy dirs
+COPY ./ ./
+
+RUN sudo chown -R rust:rust .
+RUN cargo chef prepare --recipe-path recipe.json
+
+# Cargo chef cache dependencies
+FROM $RUST_BUILDER_IMAGE as cacher
+ARG CARGO_BUILD_TARGET=x86_64-unknown-linux-musl
+WORKDIR /app
+RUN cargo install cargo-chef --version 0.1.6
+COPY --from=planner /app/recipe.json ./recipe.json
+RUN sudo chown -R rust:rust .
+RUN cargo chef cook --target ${CARGO_BUILD_TARGET} --recipe-path recipe.json
+
+# Build the project
+FROM $RUST_BUILDER_IMAGE as builder
+
+ARG CARGO_BUILD_TARGET=x86_64-unknown-linux-musl
+ARG RUSTRELEASEDIR="debug"
 
 WORKDIR /app
 
-# Copy the source folders
+# Copy over the cached dependencies
-COPY . ./
+COPY --from=cacher /app/target target
+COPY --from=cacher /home/rust/.cargo /home/rust/.cargo
 
-# Build for debug
+# Copy the rest of the dirs
-RUN --mount=type=cache,target=/usr/local/cargo/registry \
+COPY ./ ./
-    --mount=type=cache,target=/app/target \
-    cargo build
-RUN --mount=type=cache,target=/app/target \
-    cp target/debug/lemmy_server lemmy_server
 
-FROM peaceiris/mdbook:v0.3.7 as docs
+RUN sudo chown -R rust:rust .
+RUN cargo build
+
+# reduce binary size
+RUN strip ./target/$CARGO_BUILD_TARGET/$RUSTRELEASEDIR/lemmy_server
+
+RUN cp ./target/$CARGO_BUILD_TARGET/$RUSTRELEASEDIR/lemmy_server /app/lemmy_server
+
+# Build the docs
+FROM $RUST_BUILDER_IMAGE as docs
 WORKDIR /app
-COPY docs ./docs
+COPY --chown=rust:rust docs ./docs
 RUN mdbook build docs/
 
-FROM ubuntu:20.10
+# The alpine runner
+FROM alpine:3.12 as lemmy
 
-# Install libpq for postgres and espeak
+# Install libpq for postgres
-RUN apt-get update -y
+RUN apk add libpq
-RUN apt-get install -y libpq-dev espeak 
+
+# Install Espeak for captchas
+RUN apk add espeak
+
+RUN addgroup -g 1000 lemmy
+RUN adduser -D -s /bin/sh -u 1000 -G lemmy lemmy
 
 # Copy resources
-COPY config/defaults.hjson /config/defaults.hjson
+COPY --chown=lemmy:lemmy config/defaults.hjson /config/defaults.hjson
-COPY --from=rust /app/lemmy_server /app/lemmy
+COPY --chown=lemmy:lemmy --from=builder /app/lemmy_server /app/lemmy
-COPY --from=docs /app/docs/book/ /app/documentation/
+COPY --chown=lemmy:lemmy --from=docs /app/docs/book/ /app/documentation/
 
+RUN chown lemmy:lemmy /app/lemmy
+USER lemmy
 EXPOSE 8536
 CMD ["/app/lemmy"]

docker/dev/docker_update.sh

@@ -1,4 +1,8 @@
 #!/bin/sh
+
+# This script uses a docker file that builds with musl, and runs on linux alpine
+# Its a bit slower for development than the volume mount.
+
 set -e
 
 mkdir -p volumes/pictrs

docker/dev/docker_update_volume_mount.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+set -e
+
+# This script uses a Dockerfile that takes advantage of docker volume mounts,
+# And runs on an ubuntu image. A little faster for development than the other
+# script
+
+mkdir -p volumes/pictrs
+sudo chown -R 991:991 volumes/pictrs
+sudo docker build ../../ --file ../dev/volume_mount.dockerfile -t lemmy-dev:latest
+sudo docker-compose up -d

docker/dev/volume_mount.dockerfile

@@ -0,0 +1,35 @@
+# syntax=docker/dockerfile:experimental
+FROM rust:1.47-buster as rust
+
+ENV HOME=/home/root
+
+WORKDIR /app
+
+# Copy the source folders
+COPY . ./
+
+# Build for debug
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/app/target \
+    cargo build
+RUN --mount=type=cache,target=/app/target \
+    cp target/debug/lemmy_server lemmy_server
+
+FROM peaceiris/mdbook:v0.3.7 as docs
+WORKDIR /app
+COPY docs ./docs
+RUN mdbook build docs/
+
+FROM ubuntu:20.10
+
+# Install libpq for postgres and espeak
+RUN apt-get update -y
+RUN apt-get install -y libpq-dev espeak 
+
+# Copy resources
+COPY config/defaults.hjson /config/defaults.hjson
+COPY --from=rust /app/lemmy_server /app/lemmy
+COPY --from=docs /app/docs/book/ /app/documentation/
+
+EXPOSE 8536
+CMD ["/app/lemmy"]

docker/prod/Dockerfile

@@ -1,25 +1,55 @@
-ARG RUST_BUILDER_IMAGE=ekidd/rust-musl-builder:stable
+ARG RUST_BUILDER_IMAGE=ekidd/rust-musl-builder:1.47.0
 
-FROM $RUST_BUILDER_IMAGE as rust
+# Cargo chef plan
+FROM $RUST_BUILDER_IMAGE as planner
+WORKDIR /app
+RUN cargo install cargo-chef --version 0.1.6
+
+# Copy dirs
+COPY ./ ./
+
+RUN sudo chown -R rust:rust .
+RUN cargo chef prepare --recipe-path recipe.json
+
+# Cargo chef cache dependencies
+FROM $RUST_BUILDER_IMAGE as cacher
+ARG CARGO_BUILD_TARGET=x86_64-unknown-linux-musl
+WORKDIR /app
+RUN cargo install cargo-chef --version 0.1.6
+COPY --from=planner /app/recipe.json ./recipe.json
+RUN sudo chown -R rust:rust .
+RUN cargo chef cook --release --target ${CARGO_BUILD_TARGET} --recipe-path recipe.json
+
+# Build the project
+FROM $RUST_BUILDER_IMAGE as builder
 
 ARG CARGO_BUILD_TARGET=x86_64-unknown-linux-musl
 ARG RUSTRELEASEDIR="release"
 
-WORKDIR /app/server
+WORKDIR /app
+
+# Copy over the cached dependencies
+COPY --from=cacher /app/target target
+COPY --from=cacher /home/rust/.cargo /home/rust/.cargo
+
+# Copy the rest of the dirs
+COPY ./ ./
+
 RUN sudo chown -R rust:rust .
-COPY . ./
 RUN cargo build --release
 
 # reduce binary size
 RUN strip ./target/$CARGO_BUILD_TARGET/$RUSTRELEASEDIR/lemmy_server
 
-RUN cp ./target/$CARGO_BUILD_TARGET/$RUSTRELEASEDIR/lemmy_server /app/server/
+RUN cp ./target/$CARGO_BUILD_TARGET/$RUSTRELEASEDIR/lemmy_server /app/lemmy_server
 
+# Build the docs
 FROM $RUST_BUILDER_IMAGE as docs
 WORKDIR /app
 COPY --chown=rust:rust docs ./docs
 RUN mdbook build docs/
 
+# The alpine runner
 FROM alpine:3.12 as lemmy
 
 # Install libpq for postgres
@@ -33,7 +63,7 @@ RUN adduser -D -s /bin/sh -u 1000 -G lemmy lemmy
 
 # Copy resources
 COPY --chown=lemmy:lemmy config/defaults.hjson /config/defaults.hjson
-COPY --chown=lemmy:lemmy --from=rust /app/server/lemmy_server /app/lemmy
+COPY --chown=lemmy:lemmy --from=builder /app/lemmy_server /app/lemmy
 COPY --chown=lemmy:lemmy --from=docs /app/docs/book/ /app/documentation/
 
 RUN chown lemmy:lemmy /app/lemmy

docs/src/contributing_apub_api_outline.md

@@ -64,6 +64,10 @@ Receives activities from user: `Follow`, `Undo/Follow`, `Create`, `Update`, `Lik
         "https://enterprise.lemmy.ml/u/riker"
     ],
     "content": "Welcome to the default community!",
+    "source": {
+        "content": "Welcome to the default community!",
+        "mediaType": "text/markdown"
+    },
     "icon": {
         "type": "Image",
         "url": "https://enterprise.lemmy.ml/pictrs/image/Z8pFFb21cl.png"
@@ -123,7 +127,11 @@ Sends and receives activities from/to other users: `Create/Note`, `Update/Note`,
     "type": "Person",
     "preferredUsername": "picard",
     "name": "Jean-Luc Picard",
-    "summary": "The user bio",
+    "content": "The user bio",
+    "source": {
+        "content": "The user bio",
+        "mediaType": "text/markdown"
+    },
     "icon": {
         "type": "Image",
         "url": "https://enterprise.lemmy.ml/pictrs/image/DS3q0colRA.jpg"
@@ -150,7 +158,7 @@ Sends and receives activities from/to other users: `Create/Note`, `Update/Note`,
 |---|---|---|
 | `preferredUsername` | yes | Name of the actor |
 | `name` | no | The user's displayname |
-| `summary` | no | User bio |
+| `content` | no | User bio |
 | `icon` | no | The user's avatar, shown next to the username |
 | `image` | no | The user's banner, shown on top of the profile |
 | `inbox` | no | ActivityPub inbox URL |
@@ -174,6 +182,10 @@ A page with title, and optional URL and text content. The URL often leads to an
     "to": "https://voyager.lemmy.ml/c/main",
     "summary": "Test thumbnail 2",
     "content": "blub blub",
+    "source": {
+        "content": "blub blub",
+        "mediaType": "text/markdown"
+    },
     "url": "https://voyager.lemmy.ml:/pictrs/image/fzGwCsq7BJ.jpg",
     "image": {
         "type": "Image",
@@ -213,6 +225,10 @@ A reply to a post, or reply to another comment. Contains only text (including re
     "attributedTo": "https://enterprise.lemmy.ml/u/picard",
     "to": "https://enterprise.lemmy.ml/c/main",
     "content": "mmmk",
+    "source": {
+        "content": "mmmk",
+        "mediaType": "text/markdown"
+    },
     "inReplyTo": [
         "https://enterprise.lemmy.ml/post/38",
         "https://voyager.lemmy.ml/comment/73"
@@ -243,6 +259,11 @@ A direct message from one user to another. Can not include additional users. Thr
     "attributedTo": "https://enterprise.lemmy.ml/u/picard",
     "to": "https://voyager.lemmy.ml/u/janeway",
     "content": "test",
+    "source": {
+        "content": "test",
+        "mediaType": "text/markdown"
+    },
+    "mediaType": "text/markdown",
     "published": "2020-10-08T19:10:46.542820+00:00",
     "updated": "2020-10-08T20:13:52.547156+00:00"
 }

lemmy_apub/Cargo.toml

@@ -14,7 +14,7 @@ lemmy_db = { path = "../lemmy_db" }
 lemmy_structs = { path = "../lemmy_structs" }
 lemmy_websocket = { path = "../lemmy_websocket" }
 diesel = "1.4"
-activitystreams = "0.7.0-alpha.4"
+activitystreams = "0.7.0-alpha.6"
 activitystreams-ext = "0.1.0-alpha.2"
 bcrypt = "0.8"
 chrono = { version = "0.4", features = ["serde"] }

lemmy_apub/src/activities/receive/comment.rs

@@ -3,11 +3,11 @@ use crate::{
   fetcher::get_or_fetch_and_insert_comment,
   ActorType,
   FromApub,
+  NoteExt,
 };
 use activitystreams::{
   activity::{ActorAndObjectRefExt, Create, Dislike, Like, Remove, Update},
   base::ExtendsExt,
-  object::Note,
 };
 use anyhow::{anyhow, Context};
 use lemmy_db::{
@@ -27,7 +27,7 @@ pub(crate) async fn receive_create_comment(
   request_counter: &mut i32,
 ) -> Result<(), LemmyError> {
   let user = get_actor_as_user(&create, context, request_counter).await?;
-  let note = Note::from_any_base(create.object().to_owned().one().context(location_info!())?)?
+  let note = NoteExt::from_any_base(create.object().to_owned().one().context(location_info!())?)?
     .context(location_info!())?;
 
   let comment =
@@ -83,7 +83,7 @@ pub(crate) async fn receive_update_comment(
   context: &LemmyContext,
   request_counter: &mut i32,
 ) -> Result<(), LemmyError> {
-  let note = Note::from_any_base(update.object().to_owned().one().context(location_info!())?)?
+  let note = NoteExt::from_any_base(update.object().to_owned().one().context(location_info!())?)?
     .context(location_info!())?;
   let user = get_actor_as_user(&update, context, request_counter).await?;
 
@@ -140,7 +140,7 @@ pub(crate) async fn receive_like_comment(
   context: &LemmyContext,
   request_counter: &mut i32,
 ) -> Result<(), LemmyError> {
-  let note = Note::from_any_base(like.object().to_owned().one().context(location_info!())?)?
+  let note = NoteExt::from_any_base(like.object().to_owned().one().context(location_info!())?)?
     .context(location_info!())?;
   let user = get_actor_as_user(&like, context, request_counter).await?;
 
@@ -191,7 +191,7 @@ pub(crate) async fn receive_dislike_comment(
   context: &LemmyContext,
   request_counter: &mut i32,
 ) -> Result<(), LemmyError> {
-  let note = Note::from_any_base(
+  let note = NoteExt::from_any_base(
     dislike
       .object()
       .to_owned()

lemmy_apub/src/activities/receive/comment_undo.rs

@@ -2,8 +2,9 @@ use crate::{
   activities::receive::get_actor_as_user,
   fetcher::get_or_fetch_and_insert_comment,
   FromApub,
+  NoteExt,
 };
-use activitystreams::{activity::*, object::Note, prelude::*};
+use activitystreams::{activity::*, prelude::*};
 use anyhow::Context;
 use lemmy_db::{
   comment::{Comment, CommentForm, CommentLike},
@@ -20,7 +21,7 @@ pub(crate) async fn receive_undo_like_comment(
   request_counter: &mut i32,
 ) -> Result<(), LemmyError> {
   let user = get_actor_as_user(like, context, request_counter).await?;
-  let note = Note::from_any_base(like.object().to_owned().one().context(location_info!())?)?
+  let note = NoteExt::from_any_base(like.object().to_owned().one().context(location_info!())?)?
     .context(location_info!())?;
 
   let comment = CommentForm::from_apub(&note, context, None, request_counter).await?;
@@ -64,7 +65,7 @@ pub(crate) async fn receive_undo_dislike_comment(
   request_counter: &mut i32,
 ) -> Result<(), LemmyError> {
   let user = get_actor_as_user(dislike, context, request_counter).await?;
-  let note = Note::from_any_base(
+  let note = NoteExt::from_any_base(
     dislike
       .object()
       .to_owned()

lemmy_apub/src/activities/receive/private_message.rs

@@ -4,11 +4,12 @@ use crate::{
   fetcher::get_or_fetch_and_upsert_user,
   inbox::get_activity_to_and_cc,
   FromApub,
+  NoteExt,
 };
 use activitystreams::{
   activity::{ActorAndObjectRefExt, Create, Delete, Undo, Update},
   base::{AsBase, ExtendsExt},
-  object::{AsObject, Note},
+  object::AsObject,
   public,
 };
 use anyhow::{anyhow, Context};
@@ -30,7 +31,7 @@ pub(crate) async fn receive_create_private_message(
 ) -> Result<(), LemmyError> {
   check_private_message_activity_valid(&create, context, request_counter).await?;
 
-  let note = Note::from_any_base(
+  let note = NoteExt::from_any_base(
     create
       .object()
       .as_one()
@@ -79,7 +80,7 @@ pub(crate) async fn receive_update_private_message(
     .as_one()
     .context(location_info!())?
     .to_owned();
-  let note = Note::from_any_base(object)?.context(location_info!())?;
+  let note = NoteExt::from_any_base(object)?.context(location_info!())?;
 
   let private_message_form =
     PrivateMessageForm::from_apub(&note, context, Some(expected_domain), request_counter).await?;

lemmy_apub/src/fetcher.rs

@@ -3,11 +3,12 @@ use crate::{
   ActorType,
   FromApub,
   GroupExt,
+  NoteExt,
   PageExt,
   PersonExt,
   APUB_JSON_CONTENT_TYPE,
 };
-use activitystreams::{base::BaseExt, collection::OrderedCollection, object::Note, prelude::*};
+use activitystreams::{base::BaseExt, collection::OrderedCollection, prelude::*};
 use anyhow::{anyhow, Context};
 use chrono::NaiveDateTime;
 use diesel::result::Error::NotFound;
@@ -91,7 +92,7 @@ enum SearchAcceptedObjects {
   Person(Box<PersonExt>),
   Group(Box<GroupExt>),
   Page(Box<PageExt>),
-  Comment(Box<Note>),
+  Comment(Box<NoteExt>),
 }
 
 /// Attempt to parse the query as URL, and fetch an ActivityPub object from it.
@@ -488,7 +489,7 @@ pub(crate) async fn get_or_fetch_and_insert_comment(
         comment_ap_id
       );
       let comment =
-        fetch_remote_object::<Note>(context.client(), comment_ap_id, recursion_counter).await?;
+        fetch_remote_object::<NoteExt>(context.client(), comment_ap_id, recursion_counter).await?;
       let comment_form = CommentForm::from_apub(
         &comment,
         context,

lemmy_apub/src/lib.rs

@@ -18,7 +18,7 @@ use activitystreams::{
   activity::Follow,
   actor::{ApActor, Group, Person},
   base::AnyBase,
-  object::{Page, Tombstone},
+  object::{ApObject, Note, Page, Tombstone},
 };
 use activitystreams_ext::{Ext1, Ext2};
 use anyhow::{anyhow, Context};
@@ -31,11 +31,12 @@ use std::net::IpAddr;
 use url::{ParseError, Url};
 
 /// Activitystreams type for community
-type GroupExt = Ext2<ApActor<Group>, GroupExtension, PublicKeyExtension>;
+type GroupExt = Ext2<ApActor<ApObject<Group>>, GroupExtension, PublicKeyExtension>;
 /// Activitystreams type for user
-type PersonExt = Ext1<ApActor<Person>, PublicKeyExtension>;
+type PersonExt = Ext1<ApActor<ApObject<Person>>, PublicKeyExtension>;
 /// Activitystreams type for post
-type PageExt = Ext1<Page, PageExtension>;
+type PageExt = Ext1<ApObject<Page>, PageExtension>;
+type NoteExt = ApObject<Note>;
 
 pub static APUB_JSON_CONTENT_TYPE: &str = "application/activity+json";
 

lemmy_apub/src/objects/comment.rs

@@ -4,12 +4,18 @@ use crate::{
     get_or_fetch_and_insert_post,
     get_or_fetch_and_upsert_user,
   },
-  objects::{check_object_domain, create_tombstone},
+  objects::{
+    check_object_domain,
+    create_tombstone,
+    get_source_markdown_value,
+    set_content_and_source,
+  },
   FromApub,
+  NoteExt,
   ToApub,
 };
 use activitystreams::{
-  object::{kind::NoteType, Note, Tombstone},
+  object::{kind::NoteType, ApObject, Note, Tombstone},
   prelude::*,
 };
 use anyhow::Context;
@@ -32,10 +38,10 @@ use url::Url;
 
 #[async_trait::async_trait(?Send)]
 impl ToApub for Comment {
-  type ApubType = Note;
+  type ApubType = NoteExt;
 
-  async fn to_apub(&self, pool: &DbPool) -> Result<Note, LemmyError> {
+  async fn to_apub(&self, pool: &DbPool) -> Result<NoteExt, LemmyError> {
-    let mut comment = Note::new();
+    let mut comment = ApObject::new(Note::new());
 
     let creator_id = self.creator_id;
     let creator = blocking(pool, move |conn| User_::read(conn, creator_id)).await??;
@@ -63,9 +69,10 @@ impl ToApub for Comment {
       .set_published(convert_datetime(self.published))
       .set_to(community.actor_id)
       .set_many_in_reply_tos(in_reply_to_vec)
-      .set_content(self.content.to_owned())
       .set_attributed_to(creator.actor_id);
 
+    set_content_and_source(&mut comment, &self.content)?;
+
     if let Some(u) = self.updated {
       comment.set_updated(convert_datetime(u));
     }
@@ -80,13 +87,13 @@ impl ToApub for Comment {
 
 #[async_trait::async_trait(?Send)]
 impl FromApub for CommentForm {
-  type ApubType = Note;
+  type ApubType = NoteExt;
 
   /// Converts a `Note` to `CommentForm`.
   ///
   /// If the parent community, post and comment(s) are not known locally, these are also fetched.
   async fn from_apub(
-    note: &Note,
+    note: &NoteExt,
     context: &LemmyContext,
     expected_domain: Option<Url>,
     request_counter: &mut i32,
@@ -124,12 +131,8 @@ impl FromApub for CommentForm {
       }
       None => None,
     };
-    let content = note
+
-      .content()
+    let content = get_source_markdown_value(note)?.context(location_info!())?;
-      .context(location_info!())?
-      .as_single_xsd_string()
-      .context(location_info!())?
-      .to_string();
     let content_slurs_removed = remove_slurs(&content);
 
     Ok(CommentForm {

lemmy_apub/src/objects/community.rs

@@ -1,7 +1,12 @@
 use crate::{
   extensions::group_extensions::GroupExtension,
   fetcher::get_or_fetch_and_upsert_user,
-  objects::{check_object_domain, create_tombstone},
+  objects::{
+    check_object_domain,
+    create_tombstone,
+    get_source_markdown_value,
+    set_content_and_source,
+  },
   ActorType,
   FromApub,
   GroupExt,
@@ -10,7 +15,7 @@ use crate::{
 use activitystreams::{
   actor::{kind::GroupType, ApActor, Endpoints, Group},
   base::BaseExt,
-  object::{Image, Tombstone},
+  object::{ApObject, Image, Tombstone},
   prelude::*,
 };
 use activitystreams_ext::Ext2;
@@ -46,7 +51,7 @@ impl ToApub for Community {
     .await??;
     let moderators: Vec<String> = moderators.into_iter().map(|m| m.user_actor_id).collect();
 
-    let mut group = Group::new();
+    let mut group = ApObject::new(Group::new());
     group
       .set_context(activitystreams::context())
       .set_id(Url::parse(&self.actor_id)?)
@@ -58,9 +63,7 @@ impl ToApub for Community {
       group.set_updated(convert_datetime(u));
     }
     if let Some(d) = self.description.to_owned() {
-      // TODO: this should be html, also add source field with raw markdown
+      set_content_and_source(&mut group, &d)?;
-      //       -> same for post.content and others
-      group.set_content(d);
     }
 
     if let Some(icon_url) = &self.icon {
@@ -138,14 +141,9 @@ impl FromApub for CommunityForm {
       .as_xsd_string()
       .context(location_info!())?
       .to_string();
-    // TODO: should be parsed as html and tags like <script> removed (or use markdown source)
+
-    //       -> same for post.content etc
+    let description = get_source_markdown_value(group)?;
-    let description = group
+
-      .inner
-      .content()
-      .map(|s| s.as_single_xsd_string())
-      .flatten()
-      .map(|s| s.to_string());
     check_slurs(&name)?;
     check_slurs(&title)?;
     check_slurs_opt(&description)?;

lemmy_apub/src/objects/mod.rs

@@ -1,8 +1,9 @@
 use crate::check_is_apub_id_valid;
 use activitystreams::{
-  base::{AsBase, BaseExt},
+  base::{AsBase, BaseExt, ExtendsExt},
   markers::Base,
-  object::{Tombstone, TombstoneExt},
+  mime::{FromStrError, Mime},
+  object::{ApObjectExt, Object, ObjectExt, Tombstone, TombstoneExt},
 };
 use anyhow::{anyhow, Context};
 use chrono::NaiveDateTime;
@@ -58,3 +59,73 @@ where
   };
   Ok(actor_id.to_string())
 }
+
+pub(in crate::objects) fn set_content_and_source<T, Kind1, Kind2>(
+  object: &mut T,
+  markdown_text: &str,
+) -> Result<(), LemmyError>
+where
+  T: ApObjectExt<Kind1> + ObjectExt<Kind2> + AsBase<Kind2>,
+{
+  let mut source = Object::<()>::new_none_type();
+  source
+    .set_content(markdown_text)
+    .set_media_type(mime_markdown()?);
+  object.set_source(source.into_any_base()?);
+
+  // set `content` to markdown for compatibility with older Lemmy versions
+  // TODO: change this to HTML in a while
+  object.set_content(markdown_text);
+  object.set_media_type(mime_markdown()?);
+  //object.set_content(markdown_to_html(markdown_text));
+  Ok(())
+}
+
+pub(in crate::objects) fn get_source_markdown_value<T, Kind1, Kind2>(
+  object: &T,
+) -> Result<Option<String>, LemmyError>
+where
+  T: ApObjectExt<Kind1> + ObjectExt<Kind2> + AsBase<Kind2>,
+{
+  let content = object
+    .content()
+    .map(|s| s.as_single_xsd_string())
+    .flatten()
+    .map(|s| s.to_string());
+  if content.is_some() {
+    let source = object.source();
+    // updated lemmy version, read markdown from `source.content`
+    if let Some(source) = source {
+      let source = Object::<()>::from_any_base(source.to_owned())?.context(location_info!())?;
+      check_is_markdown(source.media_type())?;
+      let source_content = source
+        .content()
+        .map(|s| s.as_single_xsd_string())
+        .flatten()
+        .context(location_info!())?
+        .to_string();
+      return Ok(Some(source_content));
+    }
+    // older lemmy version, read markdown from `content`
+    // TODO: remove this after a while
+    else {
+      return Ok(content);
+    }
+  }
+  Ok(None)
+}
+
+pub(in crate::objects) fn mime_markdown() -> Result<Mime, FromStrError> {
+  "text/markdown".parse()
+}
+
+pub(in crate::objects) fn check_is_markdown(mime: Option<&Mime>) -> Result<(), LemmyError> {
+  let mime = mime.context(location_info!())?;
+  if !mime.eq(&mime_markdown()?) {
+    Err(LemmyError::from(anyhow!(
+      "Lemmy only supports markdown content"
+    )))
+  } else {
+    Ok(())
+  }
+}

lemmy_apub/src/objects/post.rs

@@ -1,13 +1,18 @@
 use crate::{
   extensions::page_extension::PageExtension,
   fetcher::{get_or_fetch_and_upsert_community, get_or_fetch_and_upsert_user},
-  objects::{check_object_domain, create_tombstone},
+  objects::{
+    check_object_domain,
+    create_tombstone,
+    get_source_markdown_value,
+    set_content_and_source,
+  },
   FromApub,
   PageExt,
   ToApub,
 };
 use activitystreams::{
-  object::{kind::PageType, Image, Page, Tombstone},
+  object::{kind::PageType, ApObject, Image, Page, Tombstone},
   prelude::*,
 };
 use activitystreams_ext::Ext1;
@@ -35,7 +40,7 @@ impl ToApub for Post {
 
   // Turn a Lemmy post into an ActivityPub page that can be sent out over the network.
   async fn to_apub(&self, pool: &DbPool) -> Result<PageExt, LemmyError> {
-    let mut page = Page::new();
+    let mut page = ApObject::new(Page::new());
 
     let creator_id = self.creator_id;
     let creator = blocking(pool, move |conn| User_::read(conn, creator_id)).await??;
@@ -57,7 +62,7 @@ impl ToApub for Post {
       .set_attributed_to(creator.actor_id);
 
     if let Some(body) = &self.body {
-      page.set_content(body.to_owned());
+      set_content_and_source(&mut page, &body)?;
     }
 
     // TODO: hacky code because we get self.url == Some("")
@@ -162,13 +167,8 @@ impl FromApub for PostForm {
       .as_single_xsd_string()
       .context(location_info!())?
       .to_string();
-    let body = page
+    let body = get_source_markdown_value(page)?;
-      .inner
+
-      .content()
-      .as_ref()
-      .map(|c| c.as_single_xsd_string())
-      .flatten()
-      .map(|s| s.to_string());
     check_slurs(&name)?;
     let body_slurs_removed = body.map(|b| remove_slurs(&b));
     Ok(PostForm {

lemmy_apub/src/objects/private_message.rs

@@ -1,12 +1,18 @@
 use crate::{
   check_is_apub_id_valid,
   fetcher::get_or_fetch_and_upsert_user,
-  objects::{check_object_domain, create_tombstone},
+  objects::{
+    check_object_domain,
+    create_tombstone,
+    get_source_markdown_value,
+    set_content_and_source,
+  },
   FromApub,
+  NoteExt,
   ToApub,
 };
 use activitystreams::{
-  object::{kind::NoteType, Note, Tombstone},
+  object::{kind::NoteType, ApObject, Note, Tombstone},
   prelude::*,
 };
 use anyhow::Context;
@@ -23,10 +29,10 @@ use url::Url;
 
 #[async_trait::async_trait(?Send)]
 impl ToApub for PrivateMessage {
-  type ApubType = Note;
+  type ApubType = NoteExt;
 
-  async fn to_apub(&self, pool: &DbPool) -> Result<Note, LemmyError> {
+  async fn to_apub(&self, pool: &DbPool) -> Result<NoteExt, LemmyError> {
-    let mut private_message = Note::new();
+    let mut private_message = ApObject::new(Note::new());
 
     let creator_id = self.creator_id;
     let creator = blocking(pool, move |conn| User_::read(conn, creator_id)).await??;
@@ -38,10 +44,11 @@ impl ToApub for PrivateMessage {
       .set_context(activitystreams::context())
       .set_id(Url::parse(&self.ap_id.to_owned())?)
       .set_published(convert_datetime(self.published))
-      .set_content(self.content.to_owned())
       .set_to(recipient.actor_id)
       .set_attributed_to(creator.actor_id);
 
+    set_content_and_source(&mut private_message, &self.content)?;
+
     if let Some(u) = self.updated {
       private_message.set_updated(convert_datetime(u));
     }
@@ -56,10 +63,10 @@ impl ToApub for PrivateMessage {
 
 #[async_trait::async_trait(?Send)]
 impl FromApub for PrivateMessageForm {
-  type ApubType = Note;
+  type ApubType = NoteExt;
 
   async fn from_apub(
-    note: &Note,
+    note: &NoteExt,
     context: &LemmyContext,
     expected_domain: Option<Url>,
     request_counter: &mut i32,
@@ -83,15 +90,12 @@ impl FromApub for PrivateMessageForm {
     let ap_id = note.id_unchecked().context(location_info!())?.to_string();
     check_is_apub_id_valid(&Url::parse(&ap_id)?)?;
 
+    let content = get_source_markdown_value(note)?.context(location_info!())?;
+
     Ok(PrivateMessageForm {
       creator_id: creator.id,
       recipient_id: recipient.id,
-      content: note
+      content,
-        .content()
-        .context(location_info!())?
-        .as_single_xsd_string()
-        .context(location_info!())?
-        .to_string(),
       published: note.published().map(|u| u.to_owned().naive_local()),
       updated: note.updated().map(|u| u.to_owned().naive_local()),
       deleted: None,

lemmy_apub/src/objects/user.rs

@@ -1,7 +1,13 @@
-use crate::{objects::check_object_domain, ActorType, FromApub, PersonExt, ToApub};
+use crate::{
+  objects::{check_object_domain, get_source_markdown_value, set_content_and_source},
+  ActorType,
+  FromApub,
+  PersonExt,
+  ToApub,
+};
 use activitystreams::{
   actor::{ApActor, Endpoints, Person},
-  object::{Image, Tombstone},
+  object::{ApObject, Image, Tombstone},
   prelude::*,
 };
 use activitystreams_ext::Ext1;
@@ -24,7 +30,7 @@ impl ToApub for User_ {
   type ApubType = PersonExt;
 
   async fn to_apub(&self, _pool: &DbPool) -> Result<PersonExt, LemmyError> {
-    let mut person = Person::new();
+    let mut person = ApObject::new(Person::new());
     person
       .set_context(activitystreams::context())
       .set_id(Url::parse(&self.actor_id)?)
@@ -47,6 +53,9 @@ impl ToApub for User_ {
     }
 
     if let Some(bio) = &self.bio {
+      set_content_and_source(&mut person, bio)?;
+      // Also set summary for compatibility with older Lemmy versions.
+      // TODO: remove this after a while.
       person.set_summary(bio.to_owned());
     }
 
@@ -117,14 +126,8 @@ impl FromApub for UserForm {
       .map(|n| n.to_owned().xsd_string())
       .flatten();
 
-    // TODO a limit check (like the API does) might need to be done
+    let bio = get_source_markdown_value(person)?;
-    // here when we federate to other platforms. Same for preferred_username
+
-    let bio = person
-      .inner
-      .summary()
-      .map(|s| s.as_single_xsd_string())
-      .flatten()
-      .map(|s| s.to_string());
     check_slurs(&name)?;
     check_slurs_opt(&preferred_username)?;
     check_slurs_opt(&bio)?;