Force enable undetermined language (#2851)

* Force enable undetermined language

* update

* fix tests

crates/db_schema/src/impls/actor_language.rs

@@ -72,7 +72,7 @@ impl LocalUserLanguage {
     for_local_user_id: LocalUserId,
   ) -> Result<(), Error> {
     let conn = &mut get_conn(pool).await?;
-    let lang_ids = convert_update_languages(conn, language_ids).await?;
+    let mut lang_ids = convert_update_languages(conn, language_ids).await?;
 
     // No need to update if languages are unchanged
     let current = LocalUserLanguage::read(pool, for_local_user_id).await?;
@@ -80,6 +80,16 @@ impl LocalUserLanguage {
       return Ok(());
     }
 
+    // TODO: Force enable undetermined language for all users. This is necessary because many posts
+    //       don't have a language tag (e.g. those from other federated platforms), so Lemmy users
+    //       won't see them if undetermined language is disabled.
+    //       This hack can be removed once a majority of posts have language tags, or when it is
+    //       clearer for new users that they need to enable undetermined language.
+    //       See https://github.com/LemmyNet/lemmy-ui/issues/999
+    if !lang_ids.contains(&UNDETERMINED_ID) {
+      lang_ids.push(UNDETERMINED_ID);
+    }
+
     conn
       .build_transaction()
       .run(|conn| {
@@ -523,7 +533,7 @@ mod tests {
     let pool = &build_db_pool_for_tests().await;
 
     let (site, instance) = create_test_site(pool).await;
-    let test_langs = test_langs1(pool).await;
+    let mut test_langs = test_langs1(pool).await;
     SiteLanguage::update(pool, test_langs.clone(), &site)
       .await
       .unwrap();
@@ -542,7 +552,10 @@ mod tests {
     let local_user = LocalUser::create(pool, &local_user_form).await.unwrap();
     let local_user_langs1 = LocalUserLanguage::read(pool, local_user.id).await.unwrap();
 
-    // new user should be initialized with site languages
+    // new user should be initialized with site languages and undetermined
+    //test_langs.push(UNDETERMINED_ID);
+    //test_langs.sort();
+    test_langs.insert(0, UNDETERMINED_ID);
     assert_eq!(test_langs, local_user_langs1);
 
     // update user languages
@@ -551,7 +564,7 @@ mod tests {
       .await
       .unwrap();
     let local_user_langs2 = LocalUserLanguage::read(pool, local_user.id).await.unwrap();
-    assert_eq!(2, local_user_langs2.len());
+    assert_eq!(3, local_user_langs2.len());
 
     Person::delete(pool, person.id).await.unwrap();
     LocalUser::delete(pool, local_user.id).await.unwrap();
@@ -626,8 +639,7 @@ mod tests {
   async fn test_default_post_language() {
     let pool = &build_db_pool_for_tests().await;
     let (site, instance) = create_test_site(pool).await;
-    let mut test_langs = test_langs1(pool).await;
-    test_langs.push(UNDETERMINED_ID);
+    let test_langs = test_langs1(pool).await;
     let test_langs2 = test_langs2(pool).await;
 
     let community_form = CommunityInsertForm::builder()
@@ -656,7 +668,7 @@ mod tests {
       .await
       .unwrap();
 
-    // no overlap in user/community languages, so no default language for post
+    // no overlap in user/community languages, so defaults to undetermined
     let def1 = default_post_language(pool, community.id, local_user.id)
       .await
       .unwrap();

crates/db_views/src/comment_view.rs

@@ -468,6 +468,7 @@ mod tests {
       .build();
 
     let inserted_post = Post::create(pool, &new_post).await.unwrap();
+    let english_id = Language::read_id_from_code(pool, Some("en")).await.unwrap();
 
     // Create a comment tree with this hierarchy
     //       0
@@ -481,6 +482,7 @@ mod tests {
       .content("Comment 0".into())
       .creator_id(inserted_person.id)
       .post_id(inserted_post.id)
+      .language_id(english_id)
       .build();
 
     let inserted_comment_0 = Comment::create(pool, &comment_form_0, None).await.unwrap();
@@ -489,21 +491,19 @@ mod tests {
       .content("Comment 1, A test blocked comment".into())
       .creator_id(inserted_person_2.id)
       .post_id(inserted_post.id)
+      .language_id(english_id)
       .build();
 
     let inserted_comment_1 = Comment::create(pool, &comment_form_1, Some(&inserted_comment_0.path))
       .await
       .unwrap();
 
-    let finnish_id = Language::read_id_from_code(pool, Some("fi"))
-      .await
-      .unwrap()
-      .unwrap();
+    let finnish_id = Language::read_id_from_code(pool, Some("fi")).await.unwrap();
     let comment_form_2 = CommentInsertForm::builder()
       .content("Comment 2".into())
       .creator_id(inserted_person.id)
       .post_id(inserted_post.id)
-      .language_id(Some(finnish_id))
+      .language_id(finnish_id)
       .build();
 
     let inserted_comment_2 = Comment::create(pool, &comment_form_2, Some(&inserted_comment_0.path))
@@ -514,6 +514,7 @@ mod tests {
       .content("Comment 3".into())
       .creator_id(inserted_person.id)
       .post_id(inserted_post.id)
+      .language_id(english_id)
       .build();
 
     let _inserted_comment_3 =
@@ -736,7 +737,7 @@ mod tests {
       .unwrap();
     assert_eq!(5, all_languages.len());
 
-    // change user lang to finnish, should only show single finnish comment
+    // change user lang to finnish, should only show one post in finnish and one undetermined
     let finnish_id = Language::read_id_from_code(pool, Some("fi"))
       .await
       .unwrap()
@@ -744,19 +745,22 @@ mod tests {
     LocalUserLanguage::update(pool, vec![finnish_id], data.inserted_local_user.id)
       .await
       .unwrap();
-    let finnish_comment = CommentQuery::builder()
+    let finnish_comments = CommentQuery::builder()
       .pool(pool)
       .local_user(Some(&data.inserted_local_user))
       .build()
       .list()
       .await
       .unwrap();
-    assert_eq!(1, finnish_comment.len());
+    assert_eq!(2, finnish_comments.len());
+    let finnish_comment = finnish_comments
+      .iter()
+      .find(|c| c.comment.language_id == finnish_id);
+    assert!(finnish_comment.is_some());
     assert_eq!(
       data.inserted_comment_2.content,
-      finnish_comment[0].comment.content
+      finnish_comment.unwrap().comment.content
     );
-    assert_eq!(finnish_id, finnish_comment[0].comment.language_id);
 
     // now show all comments with undetermined language (which is the default value)
     LocalUserLanguage::update(pool, vec![UNDETERMINED_ID], data.inserted_local_user.id)
@@ -769,7 +773,7 @@ mod tests {
       .list()
       .await
       .unwrap();
-    assert_eq!(3, undetermined_comment.len());
+    assert_eq!(1, undetermined_comment.len());
 
     cleanup(data, pool).await;
   }
@@ -820,7 +824,7 @@ mod tests {
         local: true,
         distinguished: false,
         path: data.inserted_comment_0.clone().path,
-        language_id: LanguageId(0),
+        language_id: LanguageId(37),
       },
       creator: Person {
         id: data.inserted_person.id,

crates/db_views/src/post_view.rs

@@ -799,9 +799,11 @@ mod tests {
       .await
       .unwrap();
 
-    // only one french language post should be returned
-    assert_eq!(1, post_listing_french.len());
-    assert_eq!(french_id, post_listing_french[0].post.language_id);
+    // only one post in french and one undetermined should be returned
+    assert_eq!(2, post_listing_french.len());
+    assert!(post_listing_french
+      .iter()
+      .any(|p| p.post.language_id == french_id));
 
     LocalUserLanguage::update(
       pool,

migrations/2023-05-10-095739_force_enable_undetermined_language/up.sql

@@ -0,0 +1,4 @@
+-- force enable undetermined language for all users
+insert into local_user_language (local_user_id, language_id)
+    select id, 0 from local_user
+    on conflict (local_user_id, language_id) do nothing;