grishka/lemmy
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Move api code into workspace

Browse source
This commit is contained in:
Felix Ableitner 2020-09-23 17:01:37 +02:00
parent 8b6860881c
commit d40eb809dd
17 changed files with 358 additions and 263 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
Cargo.lock generated
View file

@ -1787,6 +1787,50 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "lemmy_api"
version = "0.1.0"
dependencies = [
"actix",
"actix-rt",
"actix-web",
"anyhow",
"async-trait",
"awc",
"background-jobs",
"base64 0.12.3",
"bcrypt",
"captcha",
"chrono",
"diesel",
"futures",
"http",
"http-signature-normalization-actix",
"itertools",
"jsonwebtoken",
"lazy_static",
"lemmy_apub",
"lemmy_db",
"lemmy_rate_limit",
"lemmy_structs",
"lemmy_utils",
"lemmy_websocket",
"log",
"openssl",
"percent-encoding",
"rand 0.7.3",
"reqwest",
"serde 1.0.116",
"serde_json",
"sha2",
"strum",
"strum_macros",
"thiserror",
"tokio",
"url",
"uuid 0.8.1",
]
[[package]]
name = "lemmy_apub"
version = "0.1.0"
@ -1887,6 +1931,7 @@ dependencies = [
"itertools",
"jsonwebtoken",
"lazy_static",
"lemmy_api",
"lemmy_apub",
"lemmy_db",
"lemmy_rate_limit",

4
Cargo.toml
View file

@ -8,6 +8,7 @@ lto = true
[workspace]
members = [
"lemmy_api",
"lemmy_apub",
"lemmy_utils",
"lemmy_db",
@ -17,12 +18,13 @@ members = [
]
[dependencies]
lemmy_api = { path = "./lemmy_api" }
lemmy_apub = { path = "./lemmy_apub" }
lemmy_utils = { path = "./lemmy_utils" }
lemmy_db = { path = "./lemmy_db" }
lemmy_structs = { path = "./lemmy_structs" }
lemmy_rate_limit = { path = "./lemmy_rate_limit" }
lemmy_websocket = { path = "./lemmy_websocket" }
lemmy_apub = { path = "./lemmy_apub" }
diesel = "1.4"
diesel_migrations = "1.4"
bcrypt = "0.8"

4
docker/prod/deploy.sh
View file

@ -13,8 +13,8 @@ third_semver=$(echo $new_tag | cut -d "." -f 3)
# Setting the version on the front end
cd ../../
# Setting the version on the backend
echo "pub const VERSION: &str = \"$new_tag\";" > "src/version.rs"
git add "src/version.rs"
echo "pub const VERSION: &str = \"$new_tag\";" > "lemmy_api/src/version.rs"
git add "lemmy_api/src/version.rs"
# Setting the version for Ansible
echo $new_tag > "ansible/VERSION"
git add "ansible/VERSION"

45
lemmy_api/Cargo.toml Normal file
View file

@ -0,0 +1,45 @@
[package]
name = "lemmy_api"
version = "0.1.0"
authors = ["Felix Ableitner <me@nutomic.com>"]
edition = "2018"
[dependencies]
lemmy_apub = { path = "../lemmy_apub" }
lemmy_utils = { path = "../lemmy_utils" }
lemmy_db = { path = "../lemmy_db" }
lemmy_structs = { path = "../lemmy_structs" }
lemmy_rate_limit = { path = "../lemmy_rate_limit" }
lemmy_websocket = { path = "../lemmy_websocket" }
diesel = "1.4"
bcrypt = "0.8"
chrono = { version = "0.4", features = ["serde"] }
serde_json = { version = "1.0", features = ["preserve_order"]}
serde = { version = "1.0", features = ["derive"] }
actix = "0.10"
actix-web = { version = "3.0", default-features = false }
actix-rt = { version = "1.1", default-features = false }
awc = { version = "2.0", default-features = false }
log = "0.4"
rand = "0.7"
strum = "0.19"
strum_macros = "0.19"
jsonwebtoken = "7.0"
lazy_static = "1.3"
url = { version = "2.1", features = ["serde"] }
percent-encoding = "2.1"
openssl = "0.10"
http = "0.2"
http-signature-normalization-actix = { version = "0.4", default-features = false, features = ["sha-2"] }
base64 = "0.12"
tokio = "0.2"
futures = "0.3"
itertools = "0.9"
uuid = { version = "0.8", features = ["serde", "v4"] }
sha2 = "0.9"
async-trait = "0.1"
captcha = "0.0"
anyhow = "1.0"
thiserror = "1.0"
background-jobs = " 0.8"
reqwest = { version = "0.10", features = ["json"] }

0
src/api/claims.rs → lemmy_api/src/claims.rs
View file

2
src/api/comment.rs → lemmy_api/src/comment.rs
View file

@ -1,4 +1,4 @@
use crate::api::{
use crate::{
check_community_ban,
get_post,
get_user_from_jwt,

2
src/api/community.rs → lemmy_api/src/community.rs
View file

@ -1,4 +1,4 @@
use crate::api::{get_user_from_jwt, get_user_from_jwt_opt, is_admin, is_mod_or_admin, Perform};
use crate::{get_user_from_jwt, get_user_from_jwt_opt, is_admin, is_mod_or_admin, Perform};
use actix_web::web::Data;
use anyhow::Context;
use lemmy_apub::ActorType;

255
src/api/mod.rs → lemmy_api/src/lib.rs
View file

@ -1,16 +1,29 @@
use crate::{api::claims::Claims, DbPool};
use crate::claims::Claims;
use actix_web::{web, web::Data};
use anyhow::anyhow;
use lemmy_db::{
community::Community,
community_view::CommunityUserBanView,
post::Post,
user::User_,
Crud,
DbPool,
};
use lemmy_structs::{blocking, comment::*, community::*, post::*, site::*, user::*};
use lemmy_utils::{APIError, ConnectionId, LemmyError};
use lemmy_utils::{
apub::get_apub_protocol_string,
request::{retry, RecvError},
settings::Settings,
APIError,
ConnectionId,
LemmyError,
};
use lemmy_websocket::{serialize_websocket_message, LemmyContext, UserOperation};
use log::error;
use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
use reqwest::Client;
use serde::Deserialize;
use std::process::Command;
pub mod claims;
pub mod comment;
@ -18,6 +31,7 @@ pub mod community;
pub mod post;
pub mod site;
pub mod user;
pub mod version;
#[async_trait::async_trait(?Send)]
pub trait Perform {
@ -30,7 +44,7 @@ pub trait Perform {
) -> Result<Self::Response, LemmyError>;
}
pub(in crate::api) async fn is_mod_or_admin(
pub(in crate) async fn is_mod_or_admin(
pool: &DbPool,
user_id: i32,
community_id: i32,
@ -52,17 +66,14 @@ pub async fn is_admin(pool: &DbPool, user_id: i32) -> Result<(), LemmyError> {
Ok(())
}
pub(in crate::api) async fn get_post(post_id: i32, pool: &DbPool) -> Result<Post, LemmyError> {
pub(in crate) async fn get_post(post_id: i32, pool: &DbPool) -> Result<Post, LemmyError> {
match blocking(pool, move |conn| Post::read(conn, post_id)).await? {
Ok(post) => Ok(post),
Err(_e) => Err(APIError::err("couldnt_find_post").into()),
}
}
pub(in crate::api) async fn get_user_from_jwt(
jwt: &str,
pool: &DbPool,
) -> Result<User_, LemmyError> {
pub(in crate) async fn get_user_from_jwt(jwt: &str, pool: &DbPool) -> Result<User_, LemmyError> {
let claims = match Claims::decode(&jwt) {
Ok(claims) => claims.claims,
Err(_e) => return Err(APIError::err("not_logged_in").into()),
@ -76,7 +87,7 @@ pub(in crate::api) async fn get_user_from_jwt(
Ok(user)
}
pub(in crate::api) async fn get_user_from_jwt_opt(
pub(in crate) async fn get_user_from_jwt_opt(
jwt: &Option<String>,
pool: &DbPool,
) -> Result<Option<User_>, LemmyError> {
@ -86,7 +97,7 @@ pub(in crate::api) async fn get_user_from_jwt_opt(
}
}
pub(in crate::api) async fn check_community_ban(
pub(in crate) async fn check_community_ban(
user_id: i32,
community_id: i32,
pool: &DbPool,
@ -272,3 +283,227 @@ where
.await?;
serialize_websocket_message(&op, &res)
}
pub(crate) fn captcha_espeak_wav_base64(captcha: &str) -> Result<String, LemmyError> {
let mut built_text = String::new();
// Building proper speech text for espeak
for mut c in captcha.chars() {
let new_str = if c.is_alphabetic() {
if c.is_lowercase() {
c.make_ascii_uppercase();
format!("lower case {} ... ", c)
} else {
c.make_ascii_uppercase();
format!("capital {} ... ", c)
}
} else {
format!("{} ...", c)
};
built_text.push_str(&new_str);
}
espeak_wav_base64(&built_text)
}
pub(crate) fn espeak_wav_base64(text: &str) -> Result<String, LemmyError> {
// Make a temp file path
let uuid = uuid::Uuid::new_v4().to_string();
let file_path = format!("/tmp/lemmy_espeak_{}.wav", &uuid);
// Write the wav file
Command::new("espeak")
.arg("-w")
.arg(&file_path)
.arg(text)
.status()?;
// Read the wav file bytes
let bytes = std::fs::read(&file_path)?;
// Delete the file
std::fs::remove_file(file_path)?;
// Convert to base64
let base64 = base64::encode(bytes);
Ok(base64)
}
#[derive(Deserialize, Debug)]
pub(crate) struct IframelyResponse {
title: Option<String>,
description: Option<String>,
thumbnail_url: Option<String>,
html: Option<String>,
}
pub(crate) async fn fetch_iframely(
client: &Client,
url: &str,
) -> Result<IframelyResponse, LemmyError> {
let fetch_url = format!("http://iframely/oembed?url={}", url);
let response = retry(|| client.get(&fetch_url).send()).await?;
let res: IframelyResponse = response
.json()
.await
.map_err(|e| RecvError(e.to_string()))?;
Ok(res)
}
#[derive(Deserialize, Debug, Clone)]
pub(crate) struct PictrsResponse {
files: Vec<PictrsFile>,
msg: String,
}
#[derive(Deserialize, Debug, Clone)]
pub(crate) struct PictrsFile {
file: String,
delete_token: String,
}
pub(crate) async fn fetch_pictrs(
client: &Client,
image_url: &str,
) -> Result<PictrsResponse, LemmyError> {
is_image_content_type(client, image_url).await?;
let fetch_url = format!(
"http://pictrs:8080/image/download?url={}",
utf8_percent_encode(image_url, NON_ALPHANUMERIC) // TODO this might not be needed
);
let response = retry(|| client.get(&fetch_url).send()).await?;
let response: PictrsResponse = response
.json()
.await
.map_err(|e| RecvError(e.to_string()))?;
if response.msg == "ok" {
Ok(response)
} else {
Err(anyhow!("{}", &response.msg).into())
}
}
async fn fetch_iframely_and_pictrs_data(
client: &Client,
url: Option<String>,
) -> (
Option<String>,
Option<String>,
Option<String>,
Option<String>,
) {
match &url {
Some(url) => {
// Fetch iframely data
let (iframely_title, iframely_description, iframely_thumbnail_url, iframely_html) =
match fetch_iframely(client, url).await {
Ok(res) => (res.title, res.description, res.thumbnail_url, res.html),
Err(e) => {
error!("iframely err: {}", e);
(None, None, None, None)
}
};
// Fetch pictrs thumbnail
let pictrs_hash = match iframely_thumbnail_url {
Some(iframely_thumbnail_url) => match fetch_pictrs(client, &iframely_thumbnail_url).await {
Ok(res) => Some(res.files[0].file.to_owned()),
Err(e) => {
error!("pictrs err: {}", e);
None
}
},
// Try to generate a small thumbnail if iframely is not supported
None => match fetch_pictrs(client, &url).await {
Ok(res) => Some(res.files[0].file.to_owned()),
Err(e) => {
error!("pictrs err: {}", e);
None
}
},
};
// The full urls are necessary for federation
let pictrs_thumbnail = if let Some(pictrs_hash) = pictrs_hash {
Some(format!(
"{}://{}/pictrs/image/{}",
get_apub_protocol_string(),
Settings::get().hostname,
pictrs_hash
))
} else {
None
};
(
iframely_title,
iframely_description,
iframely_html,
pictrs_thumbnail,
)
}
None => (None, None, None, None),
}
}
pub(crate) async fn is_image_content_type(client: &Client, test: &str) -> Result<(), LemmyError> {
let response = retry(|| client.get(test).send()).await?;
if response
.headers()
.get("Content-Type")
.ok_or_else(|| anyhow!("No Content-Type header"))?
.to_str()?
.starts_with("image/")
{
Ok(())
} else {
Err(anyhow!("Not an image type.").into())
}
}
#[cfg(test)]
mod tests {
use crate::{captcha_espeak_wav_base64, is_image_content_type};
#[test]
fn test_image() {
actix_rt::System::new("tset_image").block_on(async move {
let client = reqwest::Client::default();
assert!(is_image_content_type(&client, "https://1734811051.rsc.cdn77.org/data/images/full/365645/as-virus-kills-navajos-in-their-homes-tribal-women-provide-lifeline.jpg?w=600?w=650").await.is_ok());
assert!(is_image_content_type(&client,
"https://twitter.com/BenjaminNorton/status/1259922424272957440?s=20"
)
.await.is_err()
);
});
}
#[test]
fn test_espeak() {
assert!(captcha_espeak_wav_base64("WxRt2l").is_ok())
}
// These helped with testing
// #[test]
// fn test_iframely() {
// let res = fetch_iframely(client, "https://www.redspark.nu/?p=15341").await;
// assert!(res.is_ok());
// }
// #[test]
// fn test_pictshare() {
// let res = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpg");
// assert!(res.is_ok());
// let res_other = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpgaoeu");
// assert!(res_other.is_err());
// }
}

6
src/api/post.rs → lemmy_api/src/post.rs
View file

@ -1,6 +1,10 @@
use crate::{
api::{check_community_ban, get_user_from_jwt, get_user_from_jwt_opt, is_mod_or_admin, Perform},
check_community_ban,
fetch_iframely_and_pictrs_data,
get_user_from_jwt,
get_user_from_jwt_opt,
is_mod_or_admin,
Perform,
};
use actix_web::web::Data;
use lemmy_apub::{ApubLikeableType, ApubObjectType};

5
src/api/site.rs → lemmy_api/src/site.rs
View file

@ -1,7 +1,4 @@
use crate::{
api::{get_user_from_jwt, get_user_from_jwt_opt, is_admin, Perform},
version,
};
use crate::{get_user_from_jwt, get_user_from_jwt_opt, is_admin, version, Perform};
use actix_web::web::Data;
use anyhow::Context;
use lemmy_apub::fetcher::search_by_apub_id;

6
src/api/user.rs → lemmy_api/src/user.rs
View file

@ -1,6 +1,10 @@
use crate::{
api::{claims::Claims, get_user_from_jwt, get_user_from_jwt_opt, is_admin, Perform},
captcha_espeak_wav_base64,
claims::Claims,
get_user_from_jwt,
get_user_from_jwt_opt,
is_admin,
Perform,
};
use actix_web::web::Data;
use anyhow::Context;

0
src/version.rs → lemmy_api/src/version.rs
View file

234
src/lib.rs
View file

@ -1,238 +1,4 @@
#![recursion_limit = "512"]
pub mod api;
pub mod code_migrations;
pub mod routes;
pub mod version;
use anyhow::anyhow;
use lemmy_db::DbPool;
use lemmy_utils::{
apub::get_apub_protocol_string,
request::{retry, RecvError},
settings::Settings,
LemmyError,
};
use log::error;
use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
use reqwest::Client;
use serde::Deserialize;
use std::process::Command;
#[derive(Deserialize, Debug)]
pub struct IframelyResponse {
title: Option<String>,
description: Option<String>,
thumbnail_url: Option<String>,
html: Option<String>,
}
pub async fn fetch_iframely(client: &Client, url: &str) -> Result<IframelyResponse, LemmyError> {
let fetch_url = format!("http://iframely/oembed?url={}", url);
let response = retry(|| client.get(&fetch_url).send()).await?;
let res: IframelyResponse = response
.json()
.await
.map_err(|e| RecvError(e.to_string()))?;
Ok(res)
}
#[derive(Deserialize, Debug, Clone)]
pub struct PictrsResponse {
files: Vec<PictrsFile>,
msg: String,
}
#[derive(Deserialize, Debug, Clone)]
pub struct PictrsFile {
file: String,
delete_token: String,
}
pub async fn fetch_pictrs(client: &Client, image_url: &str) -> Result<PictrsResponse, LemmyError> {
is_image_content_type(client, image_url).await?;
let fetch_url = format!(
"http://pictrs:8080/image/download?url={}",
utf8_percent_encode(image_url, NON_ALPHANUMERIC) // TODO this might not be needed
);
let response = retry(|| client.get(&fetch_url).send()).await?;
let response: PictrsResponse = response
.json()
.await
.map_err(|e| RecvError(e.to_string()))?;
if response.msg == "ok" {
Ok(response)
} else {
Err(anyhow!("{}", &response.msg).into())
}
}
async fn fetch_iframely_and_pictrs_data(
client: &Client,
url: Option<String>,
) -> (
Option<String>,
Option<String>,
Option<String>,
Option<String>,
) {
match &url {
Some(url) => {
// Fetch iframely data
let (iframely_title, iframely_description, iframely_thumbnail_url, iframely_html) =
match fetch_iframely(client, url).await {
Ok(res) => (res.title, res.description, res.thumbnail_url, res.html),
Err(e) => {
error!("iframely err: {}", e);
(None, None, None, None)
}
};
// Fetch pictrs thumbnail
let pictrs_hash = match iframely_thumbnail_url {
Some(iframely_thumbnail_url) => match fetch_pictrs(client, &iframely_thumbnail_url).await {
Ok(res) => Some(res.files[0].file.to_owned()),
Err(e) => {
error!("pictrs err: {}", e);
None
}
},
// Try to generate a small thumbnail if iframely is not supported
None => match fetch_pictrs(client, &url).await {
Ok(res) => Some(res.files[0].file.to_owned()),
Err(e) => {
error!("pictrs err: {}", e);
None
}
},
};
// The full urls are necessary for federation
let pictrs_thumbnail = if let Some(pictrs_hash) = pictrs_hash {
Some(format!(
"{}://{}/pictrs/image/{}",
get_apub_protocol_string(),
Settings::get().hostname,
pictrs_hash
))
} else {
None
};
(
iframely_title,
iframely_description,
iframely_html,
pictrs_thumbnail,
)
}
None => (None, None, None, None),
}
}
pub async fn is_image_content_type(client: &Client, test: &str) -> Result<(), LemmyError> {
let response = retry(|| client.get(test).send()).await?;
if response
.headers()
.get("Content-Type")
.ok_or_else(|| anyhow!("No Content-Type header"))?
.to_str()?
.starts_with("image/")
{
Ok(())
} else {
Err(anyhow!("Not an image type.").into())
}
}
pub fn captcha_espeak_wav_base64(captcha: &str) -> Result<String, LemmyError> {
let mut built_text = String::new();
// Building proper speech text for espeak
for mut c in captcha.chars() {
let new_str = if c.is_alphabetic() {
if c.is_lowercase() {
c.make_ascii_uppercase();
format!("lower case {} ... ", c)
} else {
c.make_ascii_uppercase();
format!("capital {} ... ", c)
}
} else {
format!("{} ...", c)
};
built_text.push_str(&new_str);
}
espeak_wav_base64(&built_text)
}
pub fn espeak_wav_base64(text: &str) -> Result<String, LemmyError> {
// Make a temp file path
let uuid = uuid::Uuid::new_v4().to_string();
let file_path = format!("/tmp/lemmy_espeak_{}.wav", &uuid);
// Write the wav file
Command::new("espeak")
.arg("-w")
.arg(&file_path)
.arg(text)
.status()?;
// Read the wav file bytes
let bytes = std::fs::read(&file_path)?;
// Delete the file
std::fs::remove_file(file_path)?;
// Convert to base64
let base64 = base64::encode(bytes);
Ok(base64)
}
#[cfg(test)]
mod tests {
use crate::{captcha_espeak_wav_base64, is_image_content_type};
#[test]
fn test_image() {
actix_rt::System::new("tset_image").block_on(async move {
let client = reqwest::Client::default();
assert!(is_image_content_type(&client, "https://1734811051.rsc.cdn77.org/data/images/full/365645/as-virus-kills-navajos-in-their-homes-tribal-women-provide-lifeline.jpg?w=600?w=650").await.is_ok());
assert!(is_image_content_type(&client,
"https://twitter.com/BenjaminNorton/status/1259922424272957440?s=20"
)
.await.is_err()
);
});
}
#[test]
fn test_espeak() {
assert!(captcha_espeak_wav_base64("WxRt2l").is_ok())
}
// These helped with testing
// #[test]
// fn test_iframely() {
// let res = fetch_iframely(client, "https://www.redspark.nu/?p=15341").await;
// assert!(res.is_ok());
// }
// #[test]
// fn test_pictshare() {
// let res = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpg");
// assert!(res.is_ok());
// let res_other = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpgaoeu");
// assert!(res_other.is_err());
// }
}

7
src/main.rs
View file

@ -16,14 +16,11 @@ use diesel::{
PgConnection,
};
use lazy_static::lazy_static;
use lemmy_api::match_websocket_operation;
use lemmy_apub::activity_queue::create_activity_queue;
use lemmy_db::get_database_url_from_env;
use lemmy_rate_limit::{rate_limiter::RateLimiter, RateLimit};
use lemmy_server::{
api::match_websocket_operation,
code_migrations::run_advanced_migrations,
routes::*,
};
use lemmy_server::{code_migrations::run_advanced_migrations, routes::*};
use lemmy_structs::blocking;
use lemmy_utils::{settings::Settings, LemmyError, CACHE_CONTROL_REGEX};
use lemmy_websocket::{chat_server::ChatServer, LemmyContext};

2
src/routes/api.rs
View file

@ -1,5 +1,5 @@
use crate::api::Perform;
use actix_web::{error::ErrorBadRequest, *};
use lemmy_api::Perform;
use lemmy_rate_limit::RateLimit;
use lemmy_structs::{comment::*, community::*, post::*, site::*, user::*};
use lemmy_websocket::LemmyContext;

2
src/routes/feeds.rs
View file

@ -1,8 +1,8 @@
use crate::api::claims::Claims;
use actix_web::{error::ErrorBadRequest, *};
use anyhow::anyhow;
use chrono::{DateTime, NaiveDateTime, Utc};
use diesel::PgConnection;
use lemmy_api::claims::Claims;
use lemmy_db::{
comment_view::{ReplyQueryBuilder, ReplyView},
community::Community,

2
src/routes/nodeinfo.rs
View file

@ -1,6 +1,6 @@
use crate::version;
use actix_web::{body::Body, error::ErrorBadRequest, *};
use anyhow::anyhow;
use lemmy_api::version;
use lemmy_db::site_view::SiteView;
use lemmy_structs::blocking;
use lemmy_utils::{apub::get_apub_protocol_string, settings::Settings, LemmyError};