LemmyNet/lemmy
2
0
Fork 0
mirror of https://github.com/LemmyNet/lemmy.git synced 2024-11-25 22:01:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

update

Browse source
This commit is contained in:
Felix Ableitner 2024-11-21 22:58:24 +01:00
parent d566710e96
commit e590a811eb
1 changed files with 11 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
crates/db_schema/src/detect_language.rs
View file

@ -1,9 +1,18 @@
use crate::{newtypes::LanguageId, source::language::Language, utils::DbPool}; use crate::{newtypes::LanguageId, source::language::Language, utils::DbPool};
use lemmy_utils::error::LemmyResult; use lemmy_utils::error::LemmyResult;
use lingua::{IsoCode639_1, Language as LinguaLanguage, LanguageDetectorBuilder}; use lingua::{IsoCode639_1, Language as LinguaLang, LanguageDetectorBuilder};
pub async fn detect_language(input: &str, pool: &mut DbPool<'_>) -> LemmyResult<LanguageId> { pub async fn detect_language(input: &str, pool: &mut DbPool<'_>) -> LemmyResult<LanguageId> {
// TODO: should only detect languages which are allowed in community // TODO: should only detect languages which are allowed in community
// TODO: cache conversion table Lingua to LanguageId and reverse (maybe load it directly from
// migration sql)
// TODO: instead of at post creation, could also run this as a background task
// TODO: probably uses a lot of ram/cpu, need to make it configurable:
// - analyze only local posts or all posts
// - low accuracy or high accuracy setting
// - min confidence value
//
// >>>> This should be a plugin!
let detector = LanguageDetectorBuilder::from_iso_codes_639_1(&[ let detector = LanguageDetectorBuilder::from_iso_codes_639_1(&[
IsoCode639_1::EN, IsoCode639_1::EN,
IsoCode639_1::ES, IsoCode639_1::ES,
@ -11,7 +20,7 @@ pub async fn detect_language(input: &str, pool: &mut DbPool<'_>) -> LemmyResult<
]) ])
.build(); .build();
let lang: Option<LinguaLanguage> = detector.detect_language_of(input); let lang: Option<LinguaLang> = detector.detect_language_of(input);
let Some(lang) = lang else { let Some(lang) = lang else {
return Ok(LanguageId(0)); return Ok(LanguageId(0));
}; };