mirror of
https://github.com/LemmyNet/lemmy.git
synced 2024-11-12 15:34:00 +00:00
Merge pull request #1152 from LemmyNet/upgrade_api_test_deps
Upgrade api test deps
This commit is contained in:
commit
10b3d9005f
9 changed files with 234 additions and 279 deletions
2
Cargo.lock
generated
2
Cargo.lock
generated
|
@ -1833,7 +1833,6 @@ dependencies = [
|
||||||
"lemmy_websocket",
|
"lemmy_websocket",
|
||||||
"log",
|
"log",
|
||||||
"openssl",
|
"openssl",
|
||||||
"percent-encoding",
|
|
||||||
"rand 0.7.3",
|
"rand 0.7.3",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"serde 1.0.116",
|
"serde 1.0.116",
|
||||||
|
@ -1984,6 +1983,7 @@ dependencies = [
|
||||||
"lettre_email",
|
"lettre_email",
|
||||||
"log",
|
"log",
|
||||||
"openssl",
|
"openssl",
|
||||||
|
"percent-encoding",
|
||||||
"rand 0.7.3",
|
"rand 0.7.3",
|
||||||
"regex",
|
"regex",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
|
|
|
@ -10,11 +10,11 @@
|
||||||
"api-test": "jest src/ -i --verbose"
|
"api-test": "jest src/ -i --verbose"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/jest": "^26.0.13",
|
"@types/jest": "^26.0.14",
|
||||||
"jest": "^26.4.2",
|
"jest": "^26.4.2",
|
||||||
"lemmy-js-client": "^1.0.11",
|
"lemmy-js-client": "^1.0.13",
|
||||||
"node-fetch": "^2.6.1",
|
"node-fetch": "^2.6.1",
|
||||||
"ts-jest": "^26.3.0",
|
"ts-jest": "^26.4.0",
|
||||||
"typescript": "^4.0.2"
|
"typescript": "^4.0.3"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -554,7 +554,7 @@
|
||||||
dependencies:
|
dependencies:
|
||||||
"@types/istanbul-lib-report" "*"
|
"@types/istanbul-lib-report" "*"
|
||||||
|
|
||||||
"@types/jest@26.x", "@types/jest@^26.0.13":
|
"@types/jest@26.x":
|
||||||
version "26.0.13"
|
version "26.0.13"
|
||||||
resolved "https://registry.yarnpkg.com/@types/jest/-/jest-26.0.13.tgz#5a7b9d5312f5dd521a38329c38ee9d3802a0b85e"
|
resolved "https://registry.yarnpkg.com/@types/jest/-/jest-26.0.13.tgz#5a7b9d5312f5dd521a38329c38ee9d3802a0b85e"
|
||||||
integrity sha512-sCzjKow4z9LILc6DhBvn5AkIfmQzDZkgtVVKmGwVrs5tuid38ws281D4l+7x1kP487+FlKDh5kfMZ8WSPAdmdA==
|
integrity sha512-sCzjKow4z9LILc6DhBvn5AkIfmQzDZkgtVVKmGwVrs5tuid38ws281D4l+7x1kP487+FlKDh5kfMZ8WSPAdmdA==
|
||||||
|
@ -562,6 +562,14 @@
|
||||||
jest-diff "^25.2.1"
|
jest-diff "^25.2.1"
|
||||||
pretty-format "^25.2.1"
|
pretty-format "^25.2.1"
|
||||||
|
|
||||||
|
"@types/jest@^26.0.14":
|
||||||
|
version "26.0.14"
|
||||||
|
resolved "https://registry.yarnpkg.com/@types/jest/-/jest-26.0.14.tgz#078695f8f65cb55c5a98450d65083b2b73e5a3f3"
|
||||||
|
integrity sha512-Hz5q8Vu0D288x3iWXePSn53W7hAjP0H7EQ6QvDO9c7t46mR0lNOLlfuwQ+JkVxuhygHzlzPX+0jKdA3ZgSh+Vg==
|
||||||
|
dependencies:
|
||||||
|
jest-diff "^25.2.1"
|
||||||
|
pretty-format "^25.2.1"
|
||||||
|
|
||||||
"@types/node@*":
|
"@types/node@*":
|
||||||
version "14.10.1"
|
version "14.10.1"
|
||||||
resolved "https://registry.yarnpkg.com/@types/node/-/node-14.10.1.tgz#cc323bad8e8a533d4822f45ce4e5326f36e42177"
|
resolved "https://registry.yarnpkg.com/@types/node/-/node-14.10.1.tgz#cc323bad8e8a533d4822f45ce4e5326f36e42177"
|
||||||
|
@ -2176,7 +2184,7 @@ jest-snapshot@^26.4.2:
|
||||||
pretty-format "^26.4.2"
|
pretty-format "^26.4.2"
|
||||||
semver "^7.3.2"
|
semver "^7.3.2"
|
||||||
|
|
||||||
jest-util@26.x, jest-util@^26.3.0:
|
jest-util@^26.1.0, jest-util@^26.3.0:
|
||||||
version "26.3.0"
|
version "26.3.0"
|
||||||
resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-26.3.0.tgz#a8974b191df30e2bf523ebbfdbaeb8efca535b3e"
|
resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-26.3.0.tgz#a8974b191df30e2bf523ebbfdbaeb8efca535b3e"
|
||||||
integrity sha512-4zpn6bwV0+AMFN0IYhH/wnzIQzRaYVrz1A8sYnRnj4UXDXbOVtWmlaZkO9mipFqZ13okIfN87aDoJWB7VH6hcw==
|
integrity sha512-4zpn6bwV0+AMFN0IYhH/wnzIQzRaYVrz1A8sYnRnj4UXDXbOVtWmlaZkO9mipFqZ13okIfN87aDoJWB7VH6hcw==
|
||||||
|
@ -2352,10 +2360,10 @@ kleur@^3.0.3:
|
||||||
resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e"
|
resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e"
|
||||||
integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==
|
integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==
|
||||||
|
|
||||||
lemmy-js-client@^1.0.11:
|
lemmy-js-client@^1.0.13:
|
||||||
version "1.0.11"
|
version "1.0.13"
|
||||||
resolved "https://registry.yarnpkg.com/lemmy-js-client/-/lemmy-js-client-1.0.11.tgz#f6ccdd5f4bf60c9ec49a4337c92d91933c0d00c2"
|
resolved "https://registry.yarnpkg.com/lemmy-js-client/-/lemmy-js-client-1.0.13.tgz#d0e1246129ade295faeec1fb4b2c7397d6947a19"
|
||||||
integrity sha512-bMvCKcP76YpSYhVSX0hGnhf9DQWpu7j4UQG2ektbpsmTi+yA4JiZKsLQXwgQH7hty42EHV0ZJVBNUpqlKnGFrA==
|
integrity sha512-Xz87cCswi/2pbDdApw9JIy8bDWRFGiGWO6IhehTytOAzf36dr4GYgsjTQTLjBX+s+BNYr8hE0+Sz4g9c+ynoJg==
|
||||||
|
|
||||||
leven@^3.1.0:
|
leven@^3.1.0:
|
||||||
version "3.1.0"
|
version "3.1.0"
|
||||||
|
@ -3374,22 +3382,22 @@ tr46@^2.0.2:
|
||||||
dependencies:
|
dependencies:
|
||||||
punycode "^2.1.1"
|
punycode "^2.1.1"
|
||||||
|
|
||||||
ts-jest@^26.3.0:
|
ts-jest@^26.4.0:
|
||||||
version "26.3.0"
|
version "26.4.0"
|
||||||
resolved "https://registry.yarnpkg.com/ts-jest/-/ts-jest-26.3.0.tgz#6b2845045347dce394f069bb59358253bc1338a9"
|
resolved "https://registry.yarnpkg.com/ts-jest/-/ts-jest-26.4.0.tgz#903c7827f3d3bc33efc2f91be294b164400c32e3"
|
||||||
integrity sha512-Jq2uKfx6bPd9+JDpZNMBJMdMQUC3sJ08acISj8NXlVgR2d5OqslEHOR2KHMgwymu8h50+lKIm0m0xj/ioYdW2Q==
|
integrity sha512-ofBzoCqf6Nv/PoWb/ByV3VNKy2KJSikamOBxvR3E6eVdIw10GwAXoyvMWXXjZJK2s6S27ZE8fI+JBTnGaovl6Q==
|
||||||
dependencies:
|
dependencies:
|
||||||
"@types/jest" "26.x"
|
"@types/jest" "26.x"
|
||||||
bs-logger "0.x"
|
bs-logger "0.x"
|
||||||
buffer-from "1.x"
|
buffer-from "1.x"
|
||||||
fast-json-stable-stringify "2.x"
|
fast-json-stable-stringify "2.x"
|
||||||
jest-util "26.x"
|
jest-util "^26.1.0"
|
||||||
json5 "2.x"
|
json5 "2.x"
|
||||||
lodash.memoize "4.x"
|
lodash.memoize "4.x"
|
||||||
make-error "1.x"
|
make-error "1.x"
|
||||||
mkdirp "1.x"
|
mkdirp "1.x"
|
||||||
semver "7.x"
|
semver "7.x"
|
||||||
yargs-parser "18.x"
|
yargs-parser "20.x"
|
||||||
|
|
||||||
tunnel-agent@^0.6.0:
|
tunnel-agent@^0.6.0:
|
||||||
version "0.6.0"
|
version "0.6.0"
|
||||||
|
@ -3437,10 +3445,10 @@ typedarray-to-buffer@^3.1.5:
|
||||||
dependencies:
|
dependencies:
|
||||||
is-typedarray "^1.0.0"
|
is-typedarray "^1.0.0"
|
||||||
|
|
||||||
typescript@^4.0.2:
|
typescript@^4.0.3:
|
||||||
version "4.0.2"
|
version "4.0.3"
|
||||||
resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.0.2.tgz#7ea7c88777c723c681e33bf7988be5d008d05ac2"
|
resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.0.3.tgz#153bbd468ef07725c1df9c77e8b453f8d36abba5"
|
||||||
integrity sha512-e4ERvRV2wb+rRZ/IQeb3jm2VxBsirQLpQhdxplZ2MEzGvDkkMmPglecnNDfSUBivMjP93vRbngYYDQqQ/78bcQ==
|
integrity sha512-tEu6DGxGgRJPb/mVPIZ48e69xCn2yRmCgYmDugAVwmJ6o+0u1RI18eO7E7WBTLYLaEVVOhwQmcdhQHweux/WPg==
|
||||||
|
|
||||||
union-value@^1.0.0:
|
union-value@^1.0.0:
|
||||||
version "1.0.1"
|
version "1.0.1"
|
||||||
|
@ -3633,7 +3641,12 @@ y18n@^4.0.0:
|
||||||
resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.0.tgz#95ef94f85ecc81d007c264e190a120f0a3c8566b"
|
resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.0.tgz#95ef94f85ecc81d007c264e190a120f0a3c8566b"
|
||||||
integrity sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==
|
integrity sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==
|
||||||
|
|
||||||
yargs-parser@18.x, yargs-parser@^18.1.2:
|
yargs-parser@20.x:
|
||||||
|
version "20.2.0"
|
||||||
|
resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.0.tgz#944791ca2be2e08ddadd3d87e9de4c6484338605"
|
||||||
|
integrity sha512-2agPoRFPoIcFzOIp6656gcvsg2ohtscpw2OINr/q46+Sq41xz2OYLqx5HRHabmFU1OARIPAYH5uteICE7mn/5A==
|
||||||
|
|
||||||
|
yargs-parser@^18.1.2:
|
||||||
version "18.1.3"
|
version "18.1.3"
|
||||||
resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-18.1.3.tgz#be68c4975c6b2abf469236b0c870362fab09a7b0"
|
resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-18.1.3.tgz#be68c4975c6b2abf469236b0c870362fab09a7b0"
|
||||||
integrity sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==
|
integrity sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==
|
||||||
|
|
|
@ -31,7 +31,6 @@ strum_macros = "0.19"
|
||||||
jsonwebtoken = "7.0"
|
jsonwebtoken = "7.0"
|
||||||
lazy_static = "1.3"
|
lazy_static = "1.3"
|
||||||
url = { version = "2.1", features = ["serde"] }
|
url = { version = "2.1", features = ["serde"] }
|
||||||
percent-encoding = "2.1"
|
|
||||||
openssl = "0.10"
|
openssl = "0.10"
|
||||||
http = "0.2"
|
http = "0.2"
|
||||||
http-signature-normalization-actix = { version = "0.4", default-features = false, features = ["sha-2"] }
|
http-signature-normalization-actix = { version = "0.4", default-features = false, features = ["sha-2"] }
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
use crate::claims::Claims;
|
use crate::claims::Claims;
|
||||||
use actix_web::{web, web::Data};
|
use actix_web::{web, web::Data};
|
||||||
use anyhow::anyhow;
|
|
||||||
use lemmy_db::{
|
use lemmy_db::{
|
||||||
community::Community,
|
community::Community,
|
||||||
community_view::CommunityUserBanView,
|
community_view::CommunityUserBanView,
|
||||||
|
@ -10,18 +9,8 @@ use lemmy_db::{
|
||||||
DbPool,
|
DbPool,
|
||||||
};
|
};
|
||||||
use lemmy_structs::{blocking, comment::*, community::*, post::*, site::*, user::*};
|
use lemmy_structs::{blocking, comment::*, community::*, post::*, site::*, user::*};
|
||||||
use lemmy_utils::{
|
use lemmy_utils::{settings::Settings, APIError, ConnectionId, LemmyError};
|
||||||
apub::get_apub_protocol_string,
|
|
||||||
request::{retry, RecvError},
|
|
||||||
settings::Settings,
|
|
||||||
APIError,
|
|
||||||
ConnectionId,
|
|
||||||
LemmyError,
|
|
||||||
};
|
|
||||||
use lemmy_websocket::{serialize_websocket_message, LemmyContext, UserOperation};
|
use lemmy_websocket::{serialize_websocket_message, LemmyContext, UserOperation};
|
||||||
use log::error;
|
|
||||||
use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
|
|
||||||
use reqwest::Client;
|
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
@ -361,179 +350,12 @@ pub(crate) fn espeak_wav_base64(text: &str) -> Result<String, LemmyError> {
|
||||||
Ok(base64)
|
Ok(base64)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize, Debug)]
|
|
||||||
pub(crate) struct IframelyResponse {
|
|
||||||
title: Option<String>,
|
|
||||||
description: Option<String>,
|
|
||||||
thumbnail_url: Option<String>,
|
|
||||||
html: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) async fn fetch_iframely(
|
|
||||||
client: &Client,
|
|
||||||
url: &str,
|
|
||||||
) -> Result<IframelyResponse, LemmyError> {
|
|
||||||
let fetch_url = format!("http://iframely/oembed?url={}", url);
|
|
||||||
|
|
||||||
let response = retry(|| client.get(&fetch_url).send()).await?;
|
|
||||||
|
|
||||||
let res: IframelyResponse = response
|
|
||||||
.json()
|
|
||||||
.await
|
|
||||||
.map_err(|e| RecvError(e.to_string()))?;
|
|
||||||
Ok(res)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Deserialize, Debug, Clone)]
|
|
||||||
pub(crate) struct PictrsResponse {
|
|
||||||
files: Vec<PictrsFile>,
|
|
||||||
msg: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Deserialize, Debug, Clone)]
|
|
||||||
pub(crate) struct PictrsFile {
|
|
||||||
file: String,
|
|
||||||
delete_token: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) async fn fetch_pictrs(
|
|
||||||
client: &Client,
|
|
||||||
image_url: &str,
|
|
||||||
) -> Result<PictrsResponse, LemmyError> {
|
|
||||||
is_image_content_type(client, image_url).await?;
|
|
||||||
|
|
||||||
let fetch_url = format!(
|
|
||||||
"http://pictrs:8080/image/download?url={}",
|
|
||||||
utf8_percent_encode(image_url, NON_ALPHANUMERIC) // TODO this might not be needed
|
|
||||||
);
|
|
||||||
|
|
||||||
let response = retry(|| client.get(&fetch_url).send()).await?;
|
|
||||||
|
|
||||||
let response: PictrsResponse = response
|
|
||||||
.json()
|
|
||||||
.await
|
|
||||||
.map_err(|e| RecvError(e.to_string()))?;
|
|
||||||
|
|
||||||
if response.msg == "ok" {
|
|
||||||
Ok(response)
|
|
||||||
} else {
|
|
||||||
Err(anyhow!("{}", &response.msg).into())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn fetch_iframely_and_pictrs_data(
|
|
||||||
client: &Client,
|
|
||||||
url: Option<String>,
|
|
||||||
) -> (
|
|
||||||
Option<String>,
|
|
||||||
Option<String>,
|
|
||||||
Option<String>,
|
|
||||||
Option<String>,
|
|
||||||
) {
|
|
||||||
match &url {
|
|
||||||
Some(url) => {
|
|
||||||
// Fetch iframely data
|
|
||||||
let (iframely_title, iframely_description, iframely_thumbnail_url, iframely_html) =
|
|
||||||
match fetch_iframely(client, url).await {
|
|
||||||
Ok(res) => (res.title, res.description, res.thumbnail_url, res.html),
|
|
||||||
Err(e) => {
|
|
||||||
error!("iframely err: {}", e);
|
|
||||||
(None, None, None, None)
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// Fetch pictrs thumbnail
|
|
||||||
let pictrs_hash = match iframely_thumbnail_url {
|
|
||||||
Some(iframely_thumbnail_url) => match fetch_pictrs(client, &iframely_thumbnail_url).await {
|
|
||||||
Ok(res) => Some(res.files[0].file.to_owned()),
|
|
||||||
Err(e) => {
|
|
||||||
error!("pictrs err: {}", e);
|
|
||||||
None
|
|
||||||
}
|
|
||||||
},
|
|
||||||
// Try to generate a small thumbnail if iframely is not supported
|
|
||||||
None => match fetch_pictrs(client, &url).await {
|
|
||||||
Ok(res) => Some(res.files[0].file.to_owned()),
|
|
||||||
Err(e) => {
|
|
||||||
error!("pictrs err: {}", e);
|
|
||||||
None
|
|
||||||
}
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
// The full urls are necessary for federation
|
|
||||||
let pictrs_thumbnail = if let Some(pictrs_hash) = pictrs_hash {
|
|
||||||
Some(format!(
|
|
||||||
"{}://{}/pictrs/image/{}",
|
|
||||||
get_apub_protocol_string(),
|
|
||||||
Settings::get().hostname,
|
|
||||||
pictrs_hash
|
|
||||||
))
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
|
|
||||||
(
|
|
||||||
iframely_title,
|
|
||||||
iframely_description,
|
|
||||||
iframely_html,
|
|
||||||
pictrs_thumbnail,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
None => (None, None, None, None),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) async fn is_image_content_type(client: &Client, test: &str) -> Result<(), LemmyError> {
|
|
||||||
let response = retry(|| client.get(test).send()).await?;
|
|
||||||
|
|
||||||
if response
|
|
||||||
.headers()
|
|
||||||
.get("Content-Type")
|
|
||||||
.ok_or_else(|| anyhow!("No Content-Type header"))?
|
|
||||||
.to_str()?
|
|
||||||
.starts_with("image/")
|
|
||||||
{
|
|
||||||
Ok(())
|
|
||||||
} else {
|
|
||||||
Err(anyhow!("Not an image type.").into())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use crate::{captcha_espeak_wav_base64, is_image_content_type};
|
use crate::captcha_espeak_wav_base64;
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_image() {
|
|
||||||
actix_rt::System::new("tset_image").block_on(async move {
|
|
||||||
let client = reqwest::Client::default();
|
|
||||||
assert!(is_image_content_type(&client, "https://1734811051.rsc.cdn77.org/data/images/full/365645/as-virus-kills-navajos-in-their-homes-tribal-women-provide-lifeline.jpg?w=600?w=650").await.is_ok());
|
|
||||||
assert!(is_image_content_type(&client,
|
|
||||||
"https://twitter.com/BenjaminNorton/status/1259922424272957440?s=20"
|
|
||||||
)
|
|
||||||
.await.is_err()
|
|
||||||
);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_espeak() {
|
fn test_espeak() {
|
||||||
assert!(captcha_espeak_wav_base64("WxRt2l").is_ok())
|
assert!(captcha_espeak_wav_base64("WxRt2l").is_ok())
|
||||||
}
|
}
|
||||||
|
|
||||||
// These helped with testing
|
|
||||||
// #[test]
|
|
||||||
// fn test_iframely() {
|
|
||||||
// let res = fetch_iframely(client, "https://www.redspark.nu/?p=15341").await;
|
|
||||||
// assert!(res.is_ok());
|
|
||||||
// }
|
|
||||||
|
|
||||||
// #[test]
|
|
||||||
// fn test_pictshare() {
|
|
||||||
// let res = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpg");
|
|
||||||
// assert!(res.is_ok());
|
|
||||||
// let res_other = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpgaoeu");
|
|
||||||
// assert!(res_other.is_err());
|
|
||||||
// }
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
use crate::{
|
use crate::{
|
||||||
check_community_ban,
|
check_community_ban,
|
||||||
fetch_iframely_and_pictrs_data,
|
|
||||||
get_user_from_jwt,
|
get_user_from_jwt,
|
||||||
get_user_from_jwt_opt,
|
get_user_from_jwt_opt,
|
||||||
is_mod_or_admin,
|
is_mod_or_admin,
|
||||||
|
@ -25,6 +24,7 @@ use lemmy_db::{
|
||||||
use lemmy_structs::{blocking, post::*};
|
use lemmy_structs::{blocking, post::*};
|
||||||
use lemmy_utils::{
|
use lemmy_utils::{
|
||||||
apub::{make_apub_endpoint, EndpointType},
|
apub::{make_apub_endpoint, EndpointType},
|
||||||
|
request::fetch_iframely_and_pictrs_data,
|
||||||
utils::{check_slurs, check_slurs_opt, is_valid_post_title},
|
utils::{check_slurs, check_slurs_opt, is_valid_post_title},
|
||||||
APIError,
|
APIError,
|
||||||
ConnectionId,
|
ConnectionId,
|
||||||
|
|
|
@ -24,7 +24,7 @@ use activitystreams::{
|
||||||
Undo,
|
Undo,
|
||||||
Update,
|
Update,
|
||||||
},
|
},
|
||||||
object::{kind::PageType, Image, Object, Page, Tombstone},
|
object::{kind::PageType, Image, Page, Tombstone},
|
||||||
prelude::*,
|
prelude::*,
|
||||||
public,
|
public,
|
||||||
};
|
};
|
||||||
|
@ -41,6 +41,7 @@ use lemmy_db::{
|
||||||
use lemmy_structs::blocking;
|
use lemmy_structs::blocking;
|
||||||
use lemmy_utils::{
|
use lemmy_utils::{
|
||||||
location_info,
|
location_info,
|
||||||
|
request::fetch_iframely_and_pictrs_data,
|
||||||
utils::{check_slurs, convert_datetime, remove_slurs},
|
utils::{check_slurs, convert_datetime, remove_slurs},
|
||||||
LemmyError,
|
LemmyError,
|
||||||
};
|
};
|
||||||
|
@ -104,24 +105,6 @@ impl ToApub for Post {
|
||||||
let url = self.url.as_ref().filter(|u| !u.is_empty());
|
let url = self.url.as_ref().filter(|u| !u.is_empty());
|
||||||
if let Some(u) = url {
|
if let Some(u) = url {
|
||||||
page.set_url(u.to_owned());
|
page.set_url(u.to_owned());
|
||||||
|
|
||||||
// Embeds
|
|
||||||
let mut page_preview = Page::new();
|
|
||||||
page_preview.set_url(u.to_owned());
|
|
||||||
|
|
||||||
if let Some(embed_title) = &self.embed_title {
|
|
||||||
page_preview.set_name(embed_title.to_owned());
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(embed_description) = &self.embed_description {
|
|
||||||
page_preview.set_summary(embed_description.to_owned());
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(embed_html) = &self.embed_html {
|
|
||||||
page_preview.set_content(embed_html.to_owned());
|
|
||||||
}
|
|
||||||
|
|
||||||
page.set_preview(page_preview.into_any_base()?);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(thumbnail_url) = &self.thumbnail_url {
|
if let Some(thumbnail_url) = &self.thumbnail_url {
|
||||||
|
@ -147,50 +130,6 @@ impl ToApub for Post {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
struct EmbedType {
|
|
||||||
title: Option<String>,
|
|
||||||
description: Option<String>,
|
|
||||||
html: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
fn extract_embed_from_apub(
|
|
||||||
page: &Ext1<Object<PageType>, PageExtension>,
|
|
||||||
) -> Result<EmbedType, LemmyError> {
|
|
||||||
match page.inner.preview() {
|
|
||||||
Some(preview) => {
|
|
||||||
let preview_page = Page::from_any_base(preview.one().context(location_info!())?.to_owned())?
|
|
||||||
.context(location_info!())?;
|
|
||||||
let title = preview_page
|
|
||||||
.name()
|
|
||||||
.map(|n| n.one())
|
|
||||||
.flatten()
|
|
||||||
.map(|s| s.as_xsd_string())
|
|
||||||
.flatten()
|
|
||||||
.map(|s| s.to_string());
|
|
||||||
let description = preview_page
|
|
||||||
.summary()
|
|
||||||
.map(|s| s.as_single_xsd_string())
|
|
||||||
.flatten()
|
|
||||||
.map(|s| s.to_string());
|
|
||||||
let html = preview_page
|
|
||||||
.content()
|
|
||||||
.map(|c| c.as_single_xsd_string())
|
|
||||||
.flatten()
|
|
||||||
.map(|s| s.to_string());
|
|
||||||
Ok(EmbedType {
|
|
||||||
title,
|
|
||||||
description,
|
|
||||||
html,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
None => Ok(EmbedType {
|
|
||||||
title: None,
|
|
||||||
description: None,
|
|
||||||
html: None,
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait::async_trait(?Send)]
|
#[async_trait::async_trait(?Send)]
|
||||||
impl FromApub for PostForm {
|
impl FromApub for PostForm {
|
||||||
type ApubType = PageExt;
|
type ApubType = PageExt;
|
||||||
|
@ -237,8 +176,19 @@ impl FromApub for PostForm {
|
||||||
.map(|u| u.to_string()),
|
.map(|u| u.to_string()),
|
||||||
None => None,
|
None => None,
|
||||||
};
|
};
|
||||||
|
let url = page
|
||||||
|
.inner
|
||||||
|
.url()
|
||||||
|
.map(|u| u.as_single_xsd_any_uri())
|
||||||
|
.flatten()
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
|
||||||
let embed = extract_embed_from_apub(page)?;
|
let (iframely_title, iframely_description, iframely_html, pictrs_thumbnail) =
|
||||||
|
if let Some(url) = &url {
|
||||||
|
fetch_iframely_and_pictrs_data(context.client(), Some(url.to_owned())).await
|
||||||
|
} else {
|
||||||
|
(None, None, None, thumbnail_url)
|
||||||
|
};
|
||||||
|
|
||||||
let name = page
|
let name = page
|
||||||
.inner
|
.inner
|
||||||
|
@ -248,12 +198,6 @@ impl FromApub for PostForm {
|
||||||
.as_single_xsd_string()
|
.as_single_xsd_string()
|
||||||
.context(location_info!())?
|
.context(location_info!())?
|
||||||
.to_string();
|
.to_string();
|
||||||
let url = page
|
|
||||||
.inner
|
|
||||||
.url()
|
|
||||||
.map(|u| u.as_single_xsd_any_uri())
|
|
||||||
.flatten()
|
|
||||||
.map(|s| s.to_string());
|
|
||||||
let body = page
|
let body = page
|
||||||
.inner
|
.inner
|
||||||
.content()
|
.content()
|
||||||
|
@ -284,10 +228,10 @@ impl FromApub for PostForm {
|
||||||
deleted: None,
|
deleted: None,
|
||||||
nsfw: ext.sensitive,
|
nsfw: ext.sensitive,
|
||||||
stickied: Some(ext.stickied),
|
stickied: Some(ext.stickied),
|
||||||
embed_title: embed.title,
|
embed_title: iframely_title,
|
||||||
embed_description: embed.description,
|
embed_description: iframely_description,
|
||||||
embed_html: embed.html,
|
embed_html: iframely_html,
|
||||||
thumbnail_url,
|
thumbnail_url: pictrs_thumbnail,
|
||||||
ap_id: Some(check_actor_domain(page, expected_domain)?),
|
ap_id: Some(check_actor_domain(page, expected_domain)?),
|
||||||
local: false,
|
local: false,
|
||||||
})
|
})
|
||||||
|
|
|
@ -16,6 +16,7 @@ lettre_email = "0.9"
|
||||||
log = "0.4"
|
log = "0.4"
|
||||||
itertools = "0.9"
|
itertools = "0.9"
|
||||||
rand = "0.7"
|
rand = "0.7"
|
||||||
|
percent-encoding = "2.1"
|
||||||
serde = { version = "1.0", features = ["derive"] }
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
serde_json = { version = "1.0", features = ["preserve_order"]}
|
serde_json = { version = "1.0", features = ["preserve_order"]}
|
||||||
thiserror = "1.0"
|
thiserror = "1.0"
|
||||||
|
|
|
@ -1,5 +1,9 @@
|
||||||
use crate::LemmyError;
|
use crate::{apub::get_apub_protocol_string, settings::Settings, LemmyError};
|
||||||
use anyhow::anyhow;
|
use anyhow::anyhow;
|
||||||
|
use log::error;
|
||||||
|
use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
|
||||||
|
use reqwest::Client;
|
||||||
|
use serde::Deserialize;
|
||||||
use std::future::Future;
|
use std::future::Future;
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
|
@ -41,3 +45,175 @@ where
|
||||||
|
|
||||||
response
|
response
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
pub(crate) struct IframelyResponse {
|
||||||
|
title: Option<String>,
|
||||||
|
description: Option<String>,
|
||||||
|
thumbnail_url: Option<String>,
|
||||||
|
html: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn fetch_iframely(
|
||||||
|
client: &Client,
|
||||||
|
url: &str,
|
||||||
|
) -> Result<IframelyResponse, LemmyError> {
|
||||||
|
let fetch_url = format!("http://iframely/oembed?url={}", url);
|
||||||
|
|
||||||
|
let response = retry(|| client.get(&fetch_url).send()).await?;
|
||||||
|
|
||||||
|
let res: IframelyResponse = response
|
||||||
|
.json()
|
||||||
|
.await
|
||||||
|
.map_err(|e| RecvError(e.to_string()))?;
|
||||||
|
Ok(res)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug, Clone)]
|
||||||
|
pub(crate) struct PictrsResponse {
|
||||||
|
files: Vec<PictrsFile>,
|
||||||
|
msg: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug, Clone)]
|
||||||
|
pub(crate) struct PictrsFile {
|
||||||
|
file: String,
|
||||||
|
delete_token: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn fetch_pictrs(
|
||||||
|
client: &Client,
|
||||||
|
image_url: &str,
|
||||||
|
) -> Result<PictrsResponse, LemmyError> {
|
||||||
|
is_image_content_type(client, image_url).await?;
|
||||||
|
|
||||||
|
let fetch_url = format!(
|
||||||
|
"http://pictrs:8080/image/download?url={}",
|
||||||
|
utf8_percent_encode(image_url, NON_ALPHANUMERIC) // TODO this might not be needed
|
||||||
|
);
|
||||||
|
|
||||||
|
let response = retry(|| client.get(&fetch_url).send()).await?;
|
||||||
|
|
||||||
|
let response: PictrsResponse = response
|
||||||
|
.json()
|
||||||
|
.await
|
||||||
|
.map_err(|e| RecvError(e.to_string()))?;
|
||||||
|
|
||||||
|
if response.msg == "ok" {
|
||||||
|
Ok(response)
|
||||||
|
} else {
|
||||||
|
Err(anyhow!("{}", &response.msg).into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn fetch_iframely_and_pictrs_data(
|
||||||
|
client: &Client,
|
||||||
|
url: Option<String>,
|
||||||
|
) -> (
|
||||||
|
Option<String>,
|
||||||
|
Option<String>,
|
||||||
|
Option<String>,
|
||||||
|
Option<String>,
|
||||||
|
) {
|
||||||
|
match &url {
|
||||||
|
Some(url) => {
|
||||||
|
// Fetch iframely data
|
||||||
|
let (iframely_title, iframely_description, iframely_thumbnail_url, iframely_html) =
|
||||||
|
match fetch_iframely(client, url).await {
|
||||||
|
Ok(res) => (res.title, res.description, res.thumbnail_url, res.html),
|
||||||
|
Err(e) => {
|
||||||
|
error!("iframely err: {}", e);
|
||||||
|
(None, None, None, None)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Fetch pictrs thumbnail
|
||||||
|
let pictrs_hash = match iframely_thumbnail_url {
|
||||||
|
Some(iframely_thumbnail_url) => match fetch_pictrs(client, &iframely_thumbnail_url).await {
|
||||||
|
Ok(res) => Some(res.files[0].file.to_owned()),
|
||||||
|
Err(e) => {
|
||||||
|
error!("pictrs err: {}", e);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
},
|
||||||
|
// Try to generate a small thumbnail if iframely is not supported
|
||||||
|
None => match fetch_pictrs(client, &url).await {
|
||||||
|
Ok(res) => Some(res.files[0].file.to_owned()),
|
||||||
|
Err(e) => {
|
||||||
|
error!("pictrs err: {}", e);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
// The full urls are necessary for federation
|
||||||
|
let pictrs_thumbnail = if let Some(pictrs_hash) = pictrs_hash {
|
||||||
|
Some(format!(
|
||||||
|
"{}://{}/pictrs/image/{}",
|
||||||
|
get_apub_protocol_string(),
|
||||||
|
Settings::get().hostname,
|
||||||
|
pictrs_hash
|
||||||
|
))
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
(
|
||||||
|
iframely_title,
|
||||||
|
iframely_description,
|
||||||
|
iframely_html,
|
||||||
|
pictrs_thumbnail,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
None => (None, None, None, None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn is_image_content_type(client: &Client, test: &str) -> Result<(), LemmyError> {
|
||||||
|
let response = retry(|| client.get(test).send()).await?;
|
||||||
|
|
||||||
|
if response
|
||||||
|
.headers()
|
||||||
|
.get("Content-Type")
|
||||||
|
.ok_or_else(|| anyhow!("No Content-Type header"))?
|
||||||
|
.to_str()?
|
||||||
|
.starts_with("image/")
|
||||||
|
{
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(anyhow!("Not an image type.").into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use crate::request::is_image_content_type;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_image() {
|
||||||
|
actix_rt::System::new("tset_image").block_on(async move {
|
||||||
|
let client = reqwest::Client::default();
|
||||||
|
assert!(is_image_content_type(&client, "https://1734811051.rsc.cdn77.org/data/images/full/365645/as-virus-kills-navajos-in-their-homes-tribal-women-provide-lifeline.jpg?w=600?w=650").await.is_ok());
|
||||||
|
assert!(is_image_content_type(&client,
|
||||||
|
"https://twitter.com/BenjaminNorton/status/1259922424272957440?s=20"
|
||||||
|
)
|
||||||
|
.await.is_err()
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// These helped with testing
|
||||||
|
// #[test]
|
||||||
|
// fn test_iframely() {
|
||||||
|
// let res = fetch_iframely(client, "https://www.redspark.nu/?p=15341").await;
|
||||||
|
// assert!(res.is_ok());
|
||||||
|
// }
|
||||||
|
|
||||||
|
// #[test]
|
||||||
|
// fn test_pictshare() {
|
||||||
|
// let res = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpg");
|
||||||
|
// assert!(res.is_ok());
|
||||||
|
// let res_other = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpgaoeu");
|
||||||
|
// assert!(res_other.is_err());
|
||||||
|
// }
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in a new issue