Changing PostLink to SiteMetadata, adding it to the API.

ansible/templates/nginx.conf

@@ -80,7 +80,7 @@ server {
     }
 
     # backend
-    location ~ ^/(api|pictrs|feeds|nodeinfo|.well-known|post_link_tags) {
+    location ~ ^/(api|pictrs|feeds|nodeinfo|.well-known) {
       proxy_pass http://0.0.0.0:{{ lemmy_port }};
       proxy_http_version 1.1;
       proxy_set_header Upgrade $http_upgrade;

crates/api/src/lib.rs

@@ -121,6 +121,9 @@ pub async fn match_websocket_operation(
     UserOperation::ResolvePostReport => {
       do_websocket_operation::<ResolvePostReport>(context, id, op, data).await
     }
+    UserOperation::GetSiteMetadata => {
+      do_websocket_operation::<GetSiteMetadata>(context, id, op, data).await
+    }
 
     // Comment ops
     UserOperation::MarkCommentAsRead => {

crates/api/src/post.rs

@@ -23,7 +23,7 @@ use lemmy_apub::{
 use lemmy_db_queries::{source::post::Post_, Crud, Likeable, Saveable};
 use lemmy_db_schema::source::{moderator::*, post::*};
 use lemmy_db_views::post_view::PostView;
-use lemmy_utils::{ApiError, ConnectionId, LemmyError};
+use lemmy_utils::{request::fetch_site_metadata, ApiError, ConnectionId, LemmyError};
 use lemmy_websocket::{send::send_post_ws_message, LemmyContext, UserOperation};
 use std::convert::TryInto;
 
@@ -285,3 +285,20 @@ impl Perform for SavePost {
     Ok(PostResponse { post_view })
   }
 }
+
+#[async_trait::async_trait(?Send)]
+impl Perform for GetSiteMetadata {
+  type Response = GetSiteMetadataResponse;
+
+  async fn perform(
+    &self,
+    context: &Data<LemmyContext>,
+    _websocket_id: Option<ConnectionId>,
+  ) -> Result<GetSiteMetadataResponse, LemmyError> {
+    let data: &Self = self;
+
+    let metadata = fetch_site_metadata(context.client(), &data.url).await?;
+
+    Ok(GetSiteMetadataResponse { metadata })
+  }
+}

crates/api_common/src/post.rs

@@ -8,6 +8,7 @@ use lemmy_db_views_actor::{
   community_moderator_view::CommunityModeratorView,
   community_view::CommunityView,
 };
+use lemmy_utils::request::SiteMetadata;
 use serde::{Deserialize, Serialize};
 use url::Url;
 
@@ -148,3 +149,13 @@ pub struct ListPostReports {
 pub struct ListPostReportsResponse {
   pub posts: Vec<PostReportView>,
 }
+
+#[derive(Deserialize, Debug)]
+pub struct GetSiteMetadata {
+  pub url: Url,
+}
+
+#[derive(Serialize, Clone, Debug)]
+pub struct GetSiteMetadataResponse {
+  pub metadata: SiteMetadata,
+}

crates/api_crud/src/post/create.rs

@@ -20,7 +20,7 @@ use lemmy_apub::{
 use lemmy_db_queries::{source::post::Post_, Crud, Likeable};
 use lemmy_db_schema::source::post::*;
 use lemmy_utils::{
-  request::fetch_post_links_and_pictrs_data,
+  request::fetch_site_metadata_and_pictrs_data,
   utils::{check_slurs, check_slurs_opt, clean_url_params, is_valid_post_title},
   ApiError,
   ConnectionId,
@@ -51,9 +51,9 @@ impl PerformCrud for CreatePost {
 
     // Fetch post links and pictrs cached image
     let data_url = data.url.as_ref();
-    let (post_links_res, pictrs_thumbnail) =
+    let (metadata_res, pictrs_thumbnail) =
-      fetch_post_links_and_pictrs_data(context.client(), data_url).await?;
+      fetch_site_metadata_and_pictrs_data(context.client(), data_url).await?;
-    let (embed_title, embed_description, embed_html) = post_links_res
+    let (embed_title, embed_description, embed_html) = metadata_res
       .map(|u| (u.title, u.description, u.html))
       .unwrap_or((None, None, None));
 

crates/api_crud/src/post/update.rs

@@ -5,7 +5,7 @@ use lemmy_apub::activities::{post::create_or_update::CreateOrUpdatePost, CreateO
 use lemmy_db_queries::{source::post::Post_, Crud};
 use lemmy_db_schema::{naive_now, source::post::*};
 use lemmy_utils::{
-  request::fetch_post_links_and_pictrs_data,
+  request::fetch_site_metadata_and_pictrs_data,
   utils::{check_slurs_opt, clean_url_params, is_valid_post_title},
   ApiError,
   ConnectionId,
@@ -51,9 +51,9 @@ impl PerformCrud for EditPost {
 
     // Fetch post links and Pictrs cached image
     let data_url = data.url.as_ref();
-    let (post_links_res, pictrs_thumbnail) =
+    let (metadata_res, pictrs_thumbnail) =
-      fetch_post_links_and_pictrs_data(context.client(), data_url).await?;
+      fetch_site_metadata_and_pictrs_data(context.client(), data_url).await?;
-    let (embed_title, embed_description, embed_html) = post_links_res
+    let (embed_title, embed_description, embed_html) = metadata_res
       .map(|u| (u.title, u.description, u.html))
       .unwrap_or((None, None, None));
 

crates/apub/src/objects/post.rs

@@ -31,7 +31,7 @@ use lemmy_db_schema::{
   },
 };
 use lemmy_utils::{
-  request::fetch_post_links_and_pictrs_data,
+  request::fetch_site_metadata_and_pictrs_data,
   utils::{check_slurs, convert_datetime, markdown_to_html, remove_slurs},
   LemmyError,
 };
@@ -188,12 +188,12 @@ impl FromApub for Post {
     let community = extract_community(&page.to, context, request_counter).await?;
 
     let thumbnail_url: Option<Url> = page.image.clone().map(|i| i.url);
-    let (post_links_res, pictrs_thumbnail) = if let Some(url) = &page.url {
+    let (metadata_res, pictrs_thumbnail) = if let Some(url) = &page.url {
-      fetch_post_links_and_pictrs_data(context.client(), Some(url)).await?
+      fetch_site_metadata_and_pictrs_data(context.client(), Some(url)).await?
     } else {
       (None, thumbnail_url)
     };
-    let (embed_title, embed_description, embed_html) = post_links_res
+    let (embed_title, embed_description, embed_html) = metadata_res
       .map(|u| (u.title, u.description, u.html))
       .unwrap_or((None, None, None));
 

crates/routes/src/lib.rs

@@ -4,5 +4,4 @@ extern crate lazy_static;
 pub mod feeds;
 pub mod images;
 pub mod nodeinfo;
-pub mod post_link_tags;
 pub mod webfinger;

crates/routes/src/post_link_tags.rs

@@ -1,29 +0,0 @@
-use actix_web::{error::ErrorBadRequest, web::Query, *};
-use anyhow::anyhow;
-use lemmy_utils::{request::fetch_post_link_tags, LemmyError};
-use lemmy_websocket::LemmyContext;
-use serde::Deserialize;
-use url::Url;
-
-#[derive(Deserialize)]
-struct Params {
-  url: String,
-}
-
-pub fn config(cfg: &mut web::ServiceConfig) {
-  cfg.route("/post_link_tags", web::get().to(get_post_links_response));
-}
-
-async fn get_post_links_response(
-  info: Query<Params>,
-  context: web::Data<LemmyContext>,
-) -> Result<HttpResponse, Error> {
-  let url =
-    Url::parse(&info.url).map_err(|_| ErrorBadRequest(LemmyError::from(anyhow!("not_found"))))?;
-
-  let json = fetch_post_link_tags(context.client(), &url)
-    .await
-    .map_err(|_| ErrorBadRequest(LemmyError::from(anyhow!("not_found"))))?;
-
-  Ok(HttpResponse::Ok().json(json))
-}

crates/utils/src/request.rs

@@ -48,16 +48,16 @@ where
   response.expect("retry http request")
 }
 
-#[derive(Deserialize, Serialize, Debug, PartialEq)]
+#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)]
-pub struct PostLinkTags {
+pub struct SiteMetadata {
   pub title: Option<String>,
   pub description: Option<String>,
-  thumbnail_url: Option<Url>,
+  image: Option<Url>,
   pub html: Option<String>,
 }
 
-/// Fetches the post link html tags (like title, description, thumbnail, etc)
+/// Fetches the post link html tags (like title, description, image, etc)
-pub async fn fetch_post_link_tags(client: &Client, url: &Url) -> Result<PostLinkTags, LemmyError> {
+pub async fn fetch_site_metadata(client: &Client, url: &Url) -> Result<SiteMetadata, LemmyError> {
   let response = retry(|| client.get(url.as_str()).send()).await?;
 
   let html = response
@@ -65,12 +65,12 @@ pub async fn fetch_post_link_tags(client: &Client, url: &Url) -> Result<PostLink
     .await
     .map_err(|e| RecvError(e.to_string()))?;
 
-  let tags = html_to_post_link_tags(&html)?;
+  let tags = html_to_site_metadata(&html)?;
 
   Ok(tags)
 }
 
-fn html_to_post_link_tags(html: &str) -> Result<PostLinkTags, LemmyError> {
+fn html_to_site_metadata(html: &str) -> Result<SiteMetadata, LemmyError> {
   let page = HTML::from_string(html.to_string(), None)?;
 
   let page_title = page.title;
@@ -95,12 +95,12 @@ fn html_to_post_link_tags(html: &str) -> Result<PostLinkTags, LemmyError> {
 
   let title = og_title.or(page_title);
   let description = og_description.or(page_description);
-  let thumbnail_url = og_image;
+  let image = og_image;
 
-  Ok(PostLinkTags {
+  Ok(SiteMetadata {
     title,
     description,
-    thumbnail_url,
+    image,
     html: None,
   })
 }
@@ -148,19 +148,21 @@ pub(crate) async fn fetch_pictrs(
 }
 
 /// Both are options, since the URL might be either an html page, or an image
-pub async fn fetch_post_links_and_pictrs_data(
+pub async fn fetch_site_metadata_and_pictrs_data(
   client: &Client,
   url: Option<&Url>,
-) -> Result<(Option<PostLinkTags>, Option<Url>), LemmyError> {
+) -> Result<(Option<SiteMetadata>, Option<Url>), LemmyError> {
   match &url {
     Some(url) => {
-      // Fetch post-links data
+      // Fetch metadata
-      let post_links_res_option = fetch_post_link_tags(client, url).await.ok();
+      // Ignore errors, since it may be an image, or not have the data.
+      // Warning, this may ignore SSL errors
+      let metadata_option = fetch_site_metadata(client, url).await.ok();
 
       // Fetch pictrs thumbnail
-      let pictrs_hash = match &post_links_res_option {
+      let pictrs_hash = match &metadata_option {
-        Some(post_link_res) => match &post_link_res.thumbnail_url {
+        Some(metadata_res) => match &metadata_res.image {
-          Some(post_links_thumbnail_url) => fetch_pictrs(client, post_links_thumbnail_url)
+          Some(metadata_image) => fetch_pictrs(client, metadata_image)
             .await?
             .map(|r| r.files[0].file.to_owned()),
           // Try to generate a small thumbnail if there's a full sized one from post-links
@@ -185,7 +187,7 @@ pub async fn fetch_post_links_and_pictrs_data(
         })
         .flatten();
 
-      Ok((post_links_res_option, pictrs_thumbnail))
+      Ok((metadata_option, pictrs_thumbnail))
     }
     None => Ok((None, None)),
   }
@@ -208,32 +210,32 @@ async fn is_image_content_type(client: &Client, test: &Url) -> Result<(), LemmyE
 
 #[cfg(test)]
 mod tests {
-  use crate::request::fetch_post_link_tags;
+  use crate::request::fetch_site_metadata;
   use url::Url;
 
-  use super::PostLinkTags;
+  use super::SiteMetadata;
 
   // These helped with testing
   #[actix_rt::test]
-  async fn test_post_links() {
+  async fn test_site_metadata() {
     let client = reqwest::Client::default();
     let sample_url = Url::parse("https://www.redspark.nu/en/peoples-war/district-leader-of-chand-led-cpn-arrested-in-bhojpur/").unwrap();
-    let sample_res = fetch_post_link_tags(&client, &sample_url).await.unwrap();
+    let sample_res = fetch_site_metadata(&client, &sample_url).await.unwrap();
     assert_eq!(
-      PostLinkTags {
+      SiteMetadata {
         title: Some("District Leader Of Chand Led CPN Arrested In Bhojpur - Redspark".to_string()),
         description: Some("BHOJPUR: A district leader of the outlawed Netra Bikram Chand alias Biplav-led outfit has been arrested. According to District Police".to_string()),
-        thumbnail_url: Some(Url::parse("https://www.redspark.nu/wp-content/uploads/2020/03/netra-bikram-chand-attends-program-1272019033653-1000x0-845x653-1.jpg").unwrap()),
+        image: Some(Url::parse("https://www.redspark.nu/wp-content/uploads/2020/03/netra-bikram-chand-attends-program-1272019033653-1000x0-845x653-1.jpg").unwrap()),
         html: None,
       }, sample_res);
 
     let youtube_url = Url::parse("https://www.youtube.com/watch?v=IquO_TcMZIQ").unwrap();
-    let youtube_res = fetch_post_link_tags(&client, &youtube_url).await.unwrap();
+    let youtube_res = fetch_site_metadata(&client, &youtube_url).await.unwrap();
     assert_eq!(
-      PostLinkTags {
+      SiteMetadata {
         title: Some("A Hard Look at Rent and Rent Seeking with Michael Hudson & Pepe Escobar".to_string()),
         description: Some("An interactive discussion on wealth inequality and the “Great Game” on the control of natural resources.In this webinar organized jointly by the Henry George...".to_string()),
-        thumbnail_url: Some(Url::parse("https://i.ytimg.com/vi/IquO_TcMZIQ/maxresdefault.jpg").unwrap()),
+        image: Some(Url::parse("https://i.ytimg.com/vi/IquO_TcMZIQ/maxresdefault.jpg").unwrap()),
         html: None,
       }, youtube_res);
   }

crates/websocket/src/lib.rs

@@ -125,6 +125,7 @@ pub enum UserOperation {
   CommunityJoin,
   ModJoin,
   ChangePassword,
+  GetSiteMetadata,
 }
 
 #[derive(EnumString, ToString, Debug, Clone)]

src/api_routes.rs

@@ -88,7 +88,11 @@ pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimit) {
             "/report/resolve",
             web::put().to(route_post::<ResolvePostReport>),
           )
-          .route("/report/list", web::get().to(route_get::<ListPostReports>)),
+          .route("/report/list", web::get().to(route_get::<ListPostReports>))
+          .route(
+            "/site_metadata",
+            web::get().to(route_get::<GetSiteMetadata>),
+          ),
       )
       // Comment
       .service(

src/main.rs

@@ -12,7 +12,7 @@ use lemmy_api_common::blocking;
 use lemmy_api_crud::match_websocket_operation_crud;
 use lemmy_apub::activity_queue::create_activity_queue;
 use lemmy_db_queries::get_database_url_from_env;
-use lemmy_routes::{feeds, images, nodeinfo, post_link_tags, webfinger};
+use lemmy_routes::{feeds, images, nodeinfo, webfinger};
 use lemmy_server::{api_routes, code_migrations::run_advanced_migrations, scheduled_tasks};
 use lemmy_utils::{
   rate_limit::{rate_limiter::RateLimiter, RateLimit},
@@ -20,6 +20,7 @@ use lemmy_utils::{
   LemmyError,
 };
 use lemmy_websocket::{chat_server::ChatServer, LemmyContext};
+use reqwest::Client;
 use std::{sync::Arc, thread};
 use tokio::sync::Mutex;
 
@@ -66,17 +67,12 @@ async fn main() -> Result<(), LemmyError> {
 
   let activity_queue = create_activity_queue();
 
-  // Required because docker installs try to use TLS, which is disabled as a reqwest feature flag
-  let client = reqwest::ClientBuilder::new()
-    .danger_accept_invalid_certs(true)
-    .build()?;
-
   let chat_server = ChatServer::startup(
     pool.clone(),
     rate_limiter.clone(),
     |c, i, o, d| Box::pin(match_websocket_operation(c, i, o, d)),
     |c, i, o, d| Box::pin(match_websocket_operation_crud(c, i, o, d)),
-    client.clone(),
+    Client::default(),
     activity_queue.clone(),
   )
   .start();
@@ -86,7 +82,7 @@ async fn main() -> Result<(), LemmyError> {
     let context = LemmyContext::create(
       pool.clone(),
       chat_server.to_owned(),
-      client.to_owned(),
+      Client::default(),
       activity_queue.to_owned(),
     );
     let rate_limiter = rate_limiter.clone();
@@ -100,7 +96,6 @@ async fn main() -> Result<(), LemmyError> {
       .configure(|cfg| images::config(cfg, &rate_limiter))
       .configure(nodeinfo::config)
       .configure(webfinger::config)
-      .configure(post_link_tags::config)
   })
   .bind((settings.bind, settings.port))?
   .run()