Merge branch 'activity-sender' into unique_ap_ids_2

server/Cargo.lock

@@ -1551,6 +1551,16 @@ dependencies = [
  "itoa",
 ]
 
+[[package]]
+name = "http-body"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b"
+dependencies = [
+ "bytes",
+ "http",
+]
+
 [[package]]
 name = "http-signature-normalization"
 version = "0.5.2"
@@ -1594,6 +1604,43 @@ dependencies = [
  "quick-error",
 ]
 
+[[package]]
+name = "hyper"
+version = "0.13.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e68a8dd9716185d9e64ea473ea6ef63529252e3e27623295a0378a19665d5eb"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "httparse",
+ "itoa",
+ "pin-project",
+ "socket2",
+ "time 0.1.43",
+ "tokio",
+ "tower-service",
+ "tracing",
+ "want",
+]
+
+[[package]]
+name = "hyper-tls"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed"
+dependencies = [
+ "bytes",
+ "hyper",
+ "native-tls",
+ "tokio",
+ "tokio-tls",
+]
+
 [[package]]
 name = "ident_case"
 version = "1.0.1"
@@ -1668,9 +1715,15 @@ dependencies = [
  "socket2",
  "widestring",
  "winapi 0.3.9",
- "winreg",
+ "winreg 0.6.2",
 ]
 
+[[package]]
+name = "ipnet"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135"
+
 [[package]]
 name = "itertools"
 version = "0.9.0"
@@ -1794,6 +1847,7 @@ dependencies = [
  "openssl",
  "percent-encoding",
  "rand 0.7.3",
+ "reqwest",
  "rss",
  "serde 1.0.114",
  "serde_json",
@@ -2720,6 +2774,42 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "reqwest"
+version = "0.10.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9eaa17ac5d7b838b7503d118fa16ad88f440498bf9ffe5424e621f93190d61e"
+dependencies = [
+ "base64 0.12.3",
+ "bytes",
+ "encoding_rs",
+ "futures-core",
+ "futures-util",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-tls",
+ "ipnet",
+ "js-sys",
+ "lazy_static",
+ "log",
+ "mime",
+ "mime_guess",
+ "native-tls",
+ "percent-encoding",
+ "pin-project-lite",
+ "serde 1.0.114",
+ "serde_json",
+ "serde_urlencoded",
+ "tokio",
+ "tokio-tls",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+ "winreg 0.7.0",
+]
+
 [[package]]
 name = "resolv-conf"
 version = "0.6.3"
@@ -3312,6 +3402,16 @@ dependencies = [
  "webpki",
 ]
 
+[[package]]
+name = "tokio-tls"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343"
+dependencies = [
+ "native-tls",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-util"
 version = "0.2.0"
@@ -3341,6 +3441,12 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tower-service"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860"
+
 [[package]]
 name = "tracing"
 version = "0.1.18"
@@ -3401,6 +3507,12 @@ dependencies = [
  "trust-dns-proto",
 ]
 
+[[package]]
+name = "try-lock"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
+
 [[package]]
 name = "twoway"
 version = "0.2.1"
@@ -3579,6 +3691,16 @@ version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed"
 
+[[package]]
+name = "want"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0"
+dependencies = [
+ "log",
+ "try-lock",
+]
+
 [[package]]
 name = "wasi"
 version = "0.9.0+wasi-snapshot-preview1"
@@ -3592,6 +3714,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f0563a9a4b071746dd5aedbc3a28c6fe9be4586fb3fbadb67c400d4f53c6b16c"
 dependencies = [
  "cfg-if",
+ "serde 1.0.114",
+ "serde_json",
  "wasm-bindgen-macro",
 ]
 
@@ -3610,6 +3734,18 @@ dependencies = [
  "wasm-bindgen-shared",
 ]
 
+[[package]]
+name = "wasm-bindgen-futures"
+version = "0.4.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95f8d235a77f880bcef268d379810ea6c0af2eacfa90b1ad5af731776e0c4699"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "wasm-bindgen",
+ "web-sys",
+]
+
 [[package]]
 name = "wasm-bindgen-macro"
 version = "0.2.67"
@@ -3726,6 +3862,15 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "winreg"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69"
+dependencies = [
+ "winapi 0.3.9",
+]
+
 [[package]]
 name = "winutil"
 version = "0.1.1"

server/Cargo.toml

@@ -54,3 +54,4 @@ captcha = "0.0.7"
 anyhow = "1.0.32"
 thiserror = "1.0.20"
 background-jobs = " 0.8.0-alpha.2"
+reqwest = { version = "0.10", features = ["json"] }

server/src/apub/fetcher.rs

@@ -15,7 +15,6 @@ use crate::{
   LemmyError,
 };
 use activitystreams::{base::BaseExt, collection::OrderedCollection, object::Note, prelude::*};
-use actix_web::client::Client;
 use anyhow::{anyhow, Context};
 use chrono::NaiveDateTime;
 use diesel::result::Error::NotFound;
@@ -35,6 +34,7 @@ use lemmy_db::{
 };
 use lemmy_utils::{get_apub_protocol_string, location_info};
 use log::debug;
+use reqwest::Client;
 use serde::Deserialize;
 use std::{fmt::Debug, time::Duration};
 use url::Url;
@@ -55,6 +55,9 @@ where
 
   let timeout = Duration::from_secs(60);
 
+  // speed up tests
+  // before: 305s
+  // after: 240s
   let json = retry(|| {
     client
       .get(url.as_str())

server/src/apub/mod.rs

@@ -31,7 +31,7 @@ use activitystreams::{
   prelude::*,
 };
 use activitystreams_ext::{Ext1, Ext2};
-use actix_web::{body::Body, client::Client, HttpResponse};
+use actix_web::{body::Body, HttpResponse};
 use anyhow::{anyhow, Context};
 use chrono::NaiveDateTime;
 use lemmy_db::{activity::do_insert_activity, user::User_};
@@ -43,6 +43,7 @@ use lemmy_utils::{
   MentionData,
 };
 use log::debug;
+use reqwest::Client;
 use serde::Serialize;
 use url::{ParseError, Url};
 
@@ -327,7 +328,7 @@ pub async fn fetch_webfinger_url(
   );
   debug!("Fetching webfinger url: {}", &fetch_url);
 
-  let mut response = retry(|| client.get(&fetch_url).send()).await?;
+  let response = retry(|| client.get(&fetch_url).send()).await?;
 
   let res: WebFingerResponse = response
     .json()

server/src/lib.rs

@@ -14,6 +14,7 @@ pub extern crate dotenv;
 pub extern crate jsonwebtoken;
 extern crate log;
 pub extern crate openssl;
+pub extern crate reqwest;
 pub extern crate rss;
 pub extern crate serde;
 pub extern crate serde_json;
@@ -35,12 +36,13 @@ use crate::{
 };
 
 use actix::Addr;
-use actix_web::{client::Client, dev::ConnectionInfo};
+use actix_web::dev::ConnectionInfo;
 use anyhow::anyhow;
 use background_jobs::QueueHandle;
 use lemmy_utils::{get_apub_protocol_string, settings::Settings};
 use log::error;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
+use reqwest::Client;
 use serde::Deserialize;
 use std::process::Command;
 
@@ -130,7 +132,7 @@ pub struct IframelyResponse {
 pub async fn fetch_iframely(client: &Client, url: &str) -> Result<IframelyResponse, LemmyError> {
   let fetch_url = format!("http://iframely/oembed?url={}", url);
 
-  let mut response = retry(|| client.get(&fetch_url).send()).await?;
+  let response = retry(|| client.get(&fetch_url).send()).await?;
 
   let res: IframelyResponse = response
     .json()
@@ -159,7 +161,7 @@ pub async fn fetch_pictrs(client: &Client, image_url: &str) -> Result<PictrsResp
     utf8_percent_encode(image_url, NON_ALPHANUMERIC) // TODO this might not be needed
   );
 
-  let mut response = retry(|| client.get(&fetch_url).send()).await?;
+  let response = retry(|| client.get(&fetch_url).send()).await?;
 
   let response: PictrsResponse = response
     .json()
@@ -332,7 +334,7 @@ mod tests {
   #[test]
   fn test_image() {
     actix_rt::System::new("tset_image").block_on(async move {
-      let client = actix_web::client::Client::default();
+      let client = reqwest::Client::default();
       assert!(is_image_content_type(&client, "https://1734811051.rsc.cdn77.org/data/images/full/365645/as-virus-kills-navajos-in-their-homes-tribal-women-provide-lifeline.jpg?w=600?w=650").await.is_ok());
       assert!(is_image_content_type(&client,
                                     "https://twitter.com/BenjaminNorton/status/1259922424272957440?s=20"

server/src/main.rs

@@ -6,7 +6,6 @@ pub extern crate lazy_static;
 use actix::prelude::*;
 use actix_web::{
   body::Body,
-  client::Client,
   dev::{Service, ServiceRequest, ServiceResponse},
   http::{
     header::{CACHE_CONTROL, CONTENT_TYPE},
@@ -30,6 +29,7 @@ use lemmy_server::{
   LemmyError,
 };
 use lemmy_utils::{settings::Settings, CACHE_CONTROL_REGEX};
+use reqwest::Client;
 use std::sync::Arc;
 use tokio::sync::Mutex;
 

server/src/request.rs

@@ -14,7 +14,7 @@ pub struct RecvError(pub String);
 pub async fn retry<F, Fut, T>(f: F) -> Result<T, LemmyError>
 where
   F: Fn() -> Fut,
-  Fut: Future<Output = Result<T, actix_web::client::SendRequestError>>,
+  Fut: Future<Output = Result<T, reqwest::Error>>,
 {
   retry_custom(|| async { Ok((f)().await) }).await
 }
@@ -22,7 +22,7 @@ where
 async fn retry_custom<F, Fut, T>(f: F) -> Result<T, LemmyError>
 where
   F: Fn() -> Fut,
-  Fut: Future<Output = Result<Result<T, actix_web::client::SendRequestError>, LemmyError>>,
+  Fut: Future<Output = Result<Result<T, reqwest::Error>, LemmyError>>,
 {
   let mut response = Err(anyhow!("connect timeout").into());
 
@@ -30,7 +30,7 @@ where
     match (f)().await? {
       Ok(t) => return Ok(t),
       Err(e) => {
-        if is_connect_timeout(&e) {
+        if e.is_timeout() {
           response = Err(SendError(e.to_string()).into());
           continue;
         }
@@ -41,13 +41,3 @@ where
 
   response
 }
-
-fn is_connect_timeout(e: &actix_web::client::SendRequestError) -> bool {
-  if let actix_web::client::SendRequestError::Connect(e) = e {
-    if let actix_web::client::ConnectError::Timeout = e {
-      return true;
-    }
-  }
-
-  false
-}

server/src/websocket/server.rs

@@ -15,11 +15,12 @@ use crate::{
   PostId,
   UserId,
 };
-use actix_web::{client::Client, web};
+use actix_web::web;
 use anyhow::Context as acontext;
 use background_jobs::QueueHandle;
 use lemmy_db::naive_now;
 use lemmy_utils::location_info;
+use reqwest::Client;
 
 /// Chat server sends this messages to session
 #[derive(Message)]