Merge pull request #1152 from LemmyNet/upgrade_api_test_deps

Upgrade api test deps
This commit is contained in:
Dessalines 2020-09-24 22:50:32 -04:00 committed by GitHub
commit 10b3d9005f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 234 additions and 279 deletions

2
Cargo.lock generated
View File

@ -1833,7 +1833,6 @@ dependencies = [
"lemmy_websocket",
"log",
"openssl",
"percent-encoding",
"rand 0.7.3",
"reqwest",
"serde 1.0.116",
@ -1984,6 +1983,7 @@ dependencies = [
"lettre_email",
"log",
"openssl",
"percent-encoding",
"rand 0.7.3",
"regex",
"reqwest",

View File

@ -10,11 +10,11 @@
"api-test": "jest src/ -i --verbose"
},
"devDependencies": {
"@types/jest": "^26.0.13",
"@types/jest": "^26.0.14",
"jest": "^26.4.2",
"lemmy-js-client": "^1.0.11",
"lemmy-js-client": "^1.0.13",
"node-fetch": "^2.6.1",
"ts-jest": "^26.3.0",
"typescript": "^4.0.2"
"ts-jest": "^26.4.0",
"typescript": "^4.0.3"
}
}

View File

@ -554,7 +554,7 @@
dependencies:
"@types/istanbul-lib-report" "*"
"@types/jest@26.x", "@types/jest@^26.0.13":
"@types/jest@26.x":
version "26.0.13"
resolved "https://registry.yarnpkg.com/@types/jest/-/jest-26.0.13.tgz#5a7b9d5312f5dd521a38329c38ee9d3802a0b85e"
integrity sha512-sCzjKow4z9LILc6DhBvn5AkIfmQzDZkgtVVKmGwVrs5tuid38ws281D4l+7x1kP487+FlKDh5kfMZ8WSPAdmdA==
@ -562,6 +562,14 @@
jest-diff "^25.2.1"
pretty-format "^25.2.1"
"@types/jest@^26.0.14":
version "26.0.14"
resolved "https://registry.yarnpkg.com/@types/jest/-/jest-26.0.14.tgz#078695f8f65cb55c5a98450d65083b2b73e5a3f3"
integrity sha512-Hz5q8Vu0D288x3iWXePSn53W7hAjP0H7EQ6QvDO9c7t46mR0lNOLlfuwQ+JkVxuhygHzlzPX+0jKdA3ZgSh+Vg==
dependencies:
jest-diff "^25.2.1"
pretty-format "^25.2.1"
"@types/node@*":
version "14.10.1"
resolved "https://registry.yarnpkg.com/@types/node/-/node-14.10.1.tgz#cc323bad8e8a533d4822f45ce4e5326f36e42177"
@ -2176,7 +2184,7 @@ jest-snapshot@^26.4.2:
pretty-format "^26.4.2"
semver "^7.3.2"
jest-util@26.x, jest-util@^26.3.0:
jest-util@^26.1.0, jest-util@^26.3.0:
version "26.3.0"
resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-26.3.0.tgz#a8974b191df30e2bf523ebbfdbaeb8efca535b3e"
integrity sha512-4zpn6bwV0+AMFN0IYhH/wnzIQzRaYVrz1A8sYnRnj4UXDXbOVtWmlaZkO9mipFqZ13okIfN87aDoJWB7VH6hcw==
@ -2352,10 +2360,10 @@ kleur@^3.0.3:
resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e"
integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==
lemmy-js-client@^1.0.11:
version "1.0.11"
resolved "https://registry.yarnpkg.com/lemmy-js-client/-/lemmy-js-client-1.0.11.tgz#f6ccdd5f4bf60c9ec49a4337c92d91933c0d00c2"
integrity sha512-bMvCKcP76YpSYhVSX0hGnhf9DQWpu7j4UQG2ektbpsmTi+yA4JiZKsLQXwgQH7hty42EHV0ZJVBNUpqlKnGFrA==
lemmy-js-client@^1.0.13:
version "1.0.13"
resolved "https://registry.yarnpkg.com/lemmy-js-client/-/lemmy-js-client-1.0.13.tgz#d0e1246129ade295faeec1fb4b2c7397d6947a19"
integrity sha512-Xz87cCswi/2pbDdApw9JIy8bDWRFGiGWO6IhehTytOAzf36dr4GYgsjTQTLjBX+s+BNYr8hE0+Sz4g9c+ynoJg==
leven@^3.1.0:
version "3.1.0"
@ -3374,22 +3382,22 @@ tr46@^2.0.2:
dependencies:
punycode "^2.1.1"
ts-jest@^26.3.0:
version "26.3.0"
resolved "https://registry.yarnpkg.com/ts-jest/-/ts-jest-26.3.0.tgz#6b2845045347dce394f069bb59358253bc1338a9"
integrity sha512-Jq2uKfx6bPd9+JDpZNMBJMdMQUC3sJ08acISj8NXlVgR2d5OqslEHOR2KHMgwymu8h50+lKIm0m0xj/ioYdW2Q==
ts-jest@^26.4.0:
version "26.4.0"
resolved "https://registry.yarnpkg.com/ts-jest/-/ts-jest-26.4.0.tgz#903c7827f3d3bc33efc2f91be294b164400c32e3"
integrity sha512-ofBzoCqf6Nv/PoWb/ByV3VNKy2KJSikamOBxvR3E6eVdIw10GwAXoyvMWXXjZJK2s6S27ZE8fI+JBTnGaovl6Q==
dependencies:
"@types/jest" "26.x"
bs-logger "0.x"
buffer-from "1.x"
fast-json-stable-stringify "2.x"
jest-util "26.x"
jest-util "^26.1.0"
json5 "2.x"
lodash.memoize "4.x"
make-error "1.x"
mkdirp "1.x"
semver "7.x"
yargs-parser "18.x"
yargs-parser "20.x"
tunnel-agent@^0.6.0:
version "0.6.0"
@ -3437,10 +3445,10 @@ typedarray-to-buffer@^3.1.5:
dependencies:
is-typedarray "^1.0.0"
typescript@^4.0.2:
version "4.0.2"
resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.0.2.tgz#7ea7c88777c723c681e33bf7988be5d008d05ac2"
integrity sha512-e4ERvRV2wb+rRZ/IQeb3jm2VxBsirQLpQhdxplZ2MEzGvDkkMmPglecnNDfSUBivMjP93vRbngYYDQqQ/78bcQ==
typescript@^4.0.3:
version "4.0.3"
resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.0.3.tgz#153bbd468ef07725c1df9c77e8b453f8d36abba5"
integrity sha512-tEu6DGxGgRJPb/mVPIZ48e69xCn2yRmCgYmDugAVwmJ6o+0u1RI18eO7E7WBTLYLaEVVOhwQmcdhQHweux/WPg==
union-value@^1.0.0:
version "1.0.1"
@ -3633,7 +3641,12 @@ y18n@^4.0.0:
resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.0.tgz#95ef94f85ecc81d007c264e190a120f0a3c8566b"
integrity sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==
yargs-parser@18.x, yargs-parser@^18.1.2:
yargs-parser@20.x:
version "20.2.0"
resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.0.tgz#944791ca2be2e08ddadd3d87e9de4c6484338605"
integrity sha512-2agPoRFPoIcFzOIp6656gcvsg2ohtscpw2OINr/q46+Sq41xz2OYLqx5HRHabmFU1OARIPAYH5uteICE7mn/5A==
yargs-parser@^18.1.2:
version "18.1.3"
resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-18.1.3.tgz#be68c4975c6b2abf469236b0c870362fab09a7b0"
integrity sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==

View File

@ -31,7 +31,6 @@ strum_macros = "0.19"
jsonwebtoken = "7.0"
lazy_static = "1.3"
url = { version = "2.1", features = ["serde"] }
percent-encoding = "2.1"
openssl = "0.10"
http = "0.2"
http-signature-normalization-actix = { version = "0.4", default-features = false, features = ["sha-2"] }

View File

@ -1,6 +1,5 @@
use crate::claims::Claims;
use actix_web::{web, web::Data};
use anyhow::anyhow;
use lemmy_db::{
community::Community,
community_view::CommunityUserBanView,
@ -10,18 +9,8 @@ use lemmy_db::{
DbPool,
};
use lemmy_structs::{blocking, comment::*, community::*, post::*, site::*, user::*};
use lemmy_utils::{
apub::get_apub_protocol_string,
request::{retry, RecvError},
settings::Settings,
APIError,
ConnectionId,
LemmyError,
};
use lemmy_utils::{settings::Settings, APIError, ConnectionId, LemmyError};
use lemmy_websocket::{serialize_websocket_message, LemmyContext, UserOperation};
use log::error;
use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
use reqwest::Client;
use serde::Deserialize;
use std::process::Command;
use url::Url;
@ -361,179 +350,12 @@ pub(crate) fn espeak_wav_base64(text: &str) -> Result<String, LemmyError> {
Ok(base64)
}
#[derive(Deserialize, Debug)]
pub(crate) struct IframelyResponse {
title: Option<String>,
description: Option<String>,
thumbnail_url: Option<String>,
html: Option<String>,
}
pub(crate) async fn fetch_iframely(
client: &Client,
url: &str,
) -> Result<IframelyResponse, LemmyError> {
let fetch_url = format!("http://iframely/oembed?url={}", url);
let response = retry(|| client.get(&fetch_url).send()).await?;
let res: IframelyResponse = response
.json()
.await
.map_err(|e| RecvError(e.to_string()))?;
Ok(res)
}
#[derive(Deserialize, Debug, Clone)]
pub(crate) struct PictrsResponse {
files: Vec<PictrsFile>,
msg: String,
}
#[derive(Deserialize, Debug, Clone)]
pub(crate) struct PictrsFile {
file: String,
delete_token: String,
}
pub(crate) async fn fetch_pictrs(
client: &Client,
image_url: &str,
) -> Result<PictrsResponse, LemmyError> {
is_image_content_type(client, image_url).await?;
let fetch_url = format!(
"http://pictrs:8080/image/download?url={}",
utf8_percent_encode(image_url, NON_ALPHANUMERIC) // TODO this might not be needed
);
let response = retry(|| client.get(&fetch_url).send()).await?;
let response: PictrsResponse = response
.json()
.await
.map_err(|e| RecvError(e.to_string()))?;
if response.msg == "ok" {
Ok(response)
} else {
Err(anyhow!("{}", &response.msg).into())
}
}
async fn fetch_iframely_and_pictrs_data(
client: &Client,
url: Option<String>,
) -> (
Option<String>,
Option<String>,
Option<String>,
Option<String>,
) {
match &url {
Some(url) => {
// Fetch iframely data
let (iframely_title, iframely_description, iframely_thumbnail_url, iframely_html) =
match fetch_iframely(client, url).await {
Ok(res) => (res.title, res.description, res.thumbnail_url, res.html),
Err(e) => {
error!("iframely err: {}", e);
(None, None, None, None)
}
};
// Fetch pictrs thumbnail
let pictrs_hash = match iframely_thumbnail_url {
Some(iframely_thumbnail_url) => match fetch_pictrs(client, &iframely_thumbnail_url).await {
Ok(res) => Some(res.files[0].file.to_owned()),
Err(e) => {
error!("pictrs err: {}", e);
None
}
},
// Try to generate a small thumbnail if iframely is not supported
None => match fetch_pictrs(client, &url).await {
Ok(res) => Some(res.files[0].file.to_owned()),
Err(e) => {
error!("pictrs err: {}", e);
None
}
},
};
// The full urls are necessary for federation
let pictrs_thumbnail = if let Some(pictrs_hash) = pictrs_hash {
Some(format!(
"{}://{}/pictrs/image/{}",
get_apub_protocol_string(),
Settings::get().hostname,
pictrs_hash
))
} else {
None
};
(
iframely_title,
iframely_description,
iframely_html,
pictrs_thumbnail,
)
}
None => (None, None, None, None),
}
}
pub(crate) async fn is_image_content_type(client: &Client, test: &str) -> Result<(), LemmyError> {
let response = retry(|| client.get(test).send()).await?;
if response
.headers()
.get("Content-Type")
.ok_or_else(|| anyhow!("No Content-Type header"))?
.to_str()?
.starts_with("image/")
{
Ok(())
} else {
Err(anyhow!("Not an image type.").into())
}
}
#[cfg(test)]
mod tests {
use crate::{captcha_espeak_wav_base64, is_image_content_type};
#[test]
fn test_image() {
actix_rt::System::new("tset_image").block_on(async move {
let client = reqwest::Client::default();
assert!(is_image_content_type(&client, "https://1734811051.rsc.cdn77.org/data/images/full/365645/as-virus-kills-navajos-in-their-homes-tribal-women-provide-lifeline.jpg?w=600?w=650").await.is_ok());
assert!(is_image_content_type(&client,
"https://twitter.com/BenjaminNorton/status/1259922424272957440?s=20"
)
.await.is_err()
);
});
}
use crate::captcha_espeak_wav_base64;
#[test]
fn test_espeak() {
assert!(captcha_espeak_wav_base64("WxRt2l").is_ok())
}
// These helped with testing
// #[test]
// fn test_iframely() {
// let res = fetch_iframely(client, "https://www.redspark.nu/?p=15341").await;
// assert!(res.is_ok());
// }
// #[test]
// fn test_pictshare() {
// let res = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpg");
// assert!(res.is_ok());
// let res_other = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpgaoeu");
// assert!(res_other.is_err());
// }
}

View File

@ -1,6 +1,5 @@
use crate::{
check_community_ban,
fetch_iframely_and_pictrs_data,
get_user_from_jwt,
get_user_from_jwt_opt,
is_mod_or_admin,
@ -25,6 +24,7 @@ use lemmy_db::{
use lemmy_structs::{blocking, post::*};
use lemmy_utils::{
apub::{make_apub_endpoint, EndpointType},
request::fetch_iframely_and_pictrs_data,
utils::{check_slurs, check_slurs_opt, is_valid_post_title},
APIError,
ConnectionId,

View File

@ -24,7 +24,7 @@ use activitystreams::{
Undo,
Update,
},
object::{kind::PageType, Image, Object, Page, Tombstone},
object::{kind::PageType, Image, Page, Tombstone},
prelude::*,
public,
};
@ -41,6 +41,7 @@ use lemmy_db::{
use lemmy_structs::blocking;
use lemmy_utils::{
location_info,
request::fetch_iframely_and_pictrs_data,
utils::{check_slurs, convert_datetime, remove_slurs},
LemmyError,
};
@ -104,24 +105,6 @@ impl ToApub for Post {
let url = self.url.as_ref().filter(|u| !u.is_empty());
if let Some(u) = url {
page.set_url(u.to_owned());
// Embeds
let mut page_preview = Page::new();
page_preview.set_url(u.to_owned());
if let Some(embed_title) = &self.embed_title {
page_preview.set_name(embed_title.to_owned());
}
if let Some(embed_description) = &self.embed_description {
page_preview.set_summary(embed_description.to_owned());
}
if let Some(embed_html) = &self.embed_html {
page_preview.set_content(embed_html.to_owned());
}
page.set_preview(page_preview.into_any_base()?);
}
if let Some(thumbnail_url) = &self.thumbnail_url {
@ -147,50 +130,6 @@ impl ToApub for Post {
}
}
struct EmbedType {
title: Option<String>,
description: Option<String>,
html: Option<String>,
}
fn extract_embed_from_apub(
page: &Ext1<Object<PageType>, PageExtension>,
) -> Result<EmbedType, LemmyError> {
match page.inner.preview() {
Some(preview) => {
let preview_page = Page::from_any_base(preview.one().context(location_info!())?.to_owned())?
.context(location_info!())?;
let title = preview_page
.name()
.map(|n| n.one())
.flatten()
.map(|s| s.as_xsd_string())
.flatten()
.map(|s| s.to_string());
let description = preview_page
.summary()
.map(|s| s.as_single_xsd_string())
.flatten()
.map(|s| s.to_string());
let html = preview_page
.content()
.map(|c| c.as_single_xsd_string())
.flatten()
.map(|s| s.to_string());
Ok(EmbedType {
title,
description,
html,
})
}
None => Ok(EmbedType {
title: None,
description: None,
html: None,
}),
}
}
#[async_trait::async_trait(?Send)]
impl FromApub for PostForm {
type ApubType = PageExt;
@ -237,8 +176,19 @@ impl FromApub for PostForm {
.map(|u| u.to_string()),
None => None,
};
let url = page
.inner
.url()
.map(|u| u.as_single_xsd_any_uri())
.flatten()
.map(|s| s.to_string());
let embed = extract_embed_from_apub(page)?;
let (iframely_title, iframely_description, iframely_html, pictrs_thumbnail) =
if let Some(url) = &url {
fetch_iframely_and_pictrs_data(context.client(), Some(url.to_owned())).await
} else {
(None, None, None, thumbnail_url)
};
let name = page
.inner
@ -248,12 +198,6 @@ impl FromApub for PostForm {
.as_single_xsd_string()
.context(location_info!())?
.to_string();
let url = page
.inner
.url()
.map(|u| u.as_single_xsd_any_uri())
.flatten()
.map(|s| s.to_string());
let body = page
.inner
.content()
@ -284,10 +228,10 @@ impl FromApub for PostForm {
deleted: None,
nsfw: ext.sensitive,
stickied: Some(ext.stickied),
embed_title: embed.title,
embed_description: embed.description,
embed_html: embed.html,
thumbnail_url,
embed_title: iframely_title,
embed_description: iframely_description,
embed_html: iframely_html,
thumbnail_url: pictrs_thumbnail,
ap_id: Some(check_actor_domain(page, expected_domain)?),
local: false,
})

View File

@ -16,6 +16,7 @@ lettre_email = "0.9"
log = "0.4"
itertools = "0.9"
rand = "0.7"
percent-encoding = "2.1"
serde = { version = "1.0", features = ["derive"] }
serde_json = { version = "1.0", features = ["preserve_order"]}
thiserror = "1.0"

View File

@ -1,5 +1,9 @@
use crate::LemmyError;
use crate::{apub::get_apub_protocol_string, settings::Settings, LemmyError};
use anyhow::anyhow;
use log::error;
use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
use reqwest::Client;
use serde::Deserialize;
use std::future::Future;
use thiserror::Error;
@ -41,3 +45,175 @@ where
response
}
#[derive(Deserialize, Debug)]
pub(crate) struct IframelyResponse {
title: Option<String>,
description: Option<String>,
thumbnail_url: Option<String>,
html: Option<String>,
}
pub(crate) async fn fetch_iframely(
client: &Client,
url: &str,
) -> Result<IframelyResponse, LemmyError> {
let fetch_url = format!("http://iframely/oembed?url={}", url);
let response = retry(|| client.get(&fetch_url).send()).await?;
let res: IframelyResponse = response
.json()
.await
.map_err(|e| RecvError(e.to_string()))?;
Ok(res)
}
#[derive(Deserialize, Debug, Clone)]
pub(crate) struct PictrsResponse {
files: Vec<PictrsFile>,
msg: String,
}
#[derive(Deserialize, Debug, Clone)]
pub(crate) struct PictrsFile {
file: String,
delete_token: String,
}
pub(crate) async fn fetch_pictrs(
client: &Client,
image_url: &str,
) -> Result<PictrsResponse, LemmyError> {
is_image_content_type(client, image_url).await?;
let fetch_url = format!(
"http://pictrs:8080/image/download?url={}",
utf8_percent_encode(image_url, NON_ALPHANUMERIC) // TODO this might not be needed
);
let response = retry(|| client.get(&fetch_url).send()).await?;
let response: PictrsResponse = response
.json()
.await
.map_err(|e| RecvError(e.to_string()))?;
if response.msg == "ok" {
Ok(response)
} else {
Err(anyhow!("{}", &response.msg).into())
}
}
pub async fn fetch_iframely_and_pictrs_data(
client: &Client,
url: Option<String>,
) -> (
Option<String>,
Option<String>,
Option<String>,
Option<String>,
) {
match &url {
Some(url) => {
// Fetch iframely data
let (iframely_title, iframely_description, iframely_thumbnail_url, iframely_html) =
match fetch_iframely(client, url).await {
Ok(res) => (res.title, res.description, res.thumbnail_url, res.html),
Err(e) => {
error!("iframely err: {}", e);
(None, None, None, None)
}
};
// Fetch pictrs thumbnail
let pictrs_hash = match iframely_thumbnail_url {
Some(iframely_thumbnail_url) => match fetch_pictrs(client, &iframely_thumbnail_url).await {
Ok(res) => Some(res.files[0].file.to_owned()),
Err(e) => {
error!("pictrs err: {}", e);
None
}
},
// Try to generate a small thumbnail if iframely is not supported
None => match fetch_pictrs(client, &url).await {
Ok(res) => Some(res.files[0].file.to_owned()),
Err(e) => {
error!("pictrs err: {}", e);
None
}
},
};
// The full urls are necessary for federation
let pictrs_thumbnail = if let Some(pictrs_hash) = pictrs_hash {
Some(format!(
"{}://{}/pictrs/image/{}",
get_apub_protocol_string(),
Settings::get().hostname,
pictrs_hash
))
} else {
None
};
(
iframely_title,
iframely_description,
iframely_html,
pictrs_thumbnail,
)
}
None => (None, None, None, None),
}
}
async fn is_image_content_type(client: &Client, test: &str) -> Result<(), LemmyError> {
let response = retry(|| client.get(test).send()).await?;
if response
.headers()
.get("Content-Type")
.ok_or_else(|| anyhow!("No Content-Type header"))?
.to_str()?
.starts_with("image/")
{
Ok(())
} else {
Err(anyhow!("Not an image type.").into())
}
}
#[cfg(test)]
mod tests {
use crate::request::is_image_content_type;
#[test]
fn test_image() {
actix_rt::System::new("tset_image").block_on(async move {
let client = reqwest::Client::default();
assert!(is_image_content_type(&client, "https://1734811051.rsc.cdn77.org/data/images/full/365645/as-virus-kills-navajos-in-their-homes-tribal-women-provide-lifeline.jpg?w=600?w=650").await.is_ok());
assert!(is_image_content_type(&client,
"https://twitter.com/BenjaminNorton/status/1259922424272957440?s=20"
)
.await.is_err()
);
});
}
// These helped with testing
// #[test]
// fn test_iframely() {
// let res = fetch_iframely(client, "https://www.redspark.nu/?p=15341").await;
// assert!(res.is_ok());
// }
// #[test]
// fn test_pictshare() {
// let res = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpg");
// assert!(res.is_ok());
// let res_other = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpgaoeu");
// assert!(res_other.is_err());
// }
}