Add controversial ranking (#3205)

* Added controversy rank property to posts and comments, and ability to sort by it

* Triggers instead of schedules tasks, integer -> double, TODO: comments don't seem to get updated with floats, divide SortTypes

* Created PersonSortType

* PersonSortType::MostComments case

* Removed unused PartialOrd trait

* Added new person sort type mappings

* SortType -> PersonSortType

* fixes

* cargo fmt

* fixes after merge with main

* Fixed bug in controversy rank trigger, removed TopX sorts from PersonSortType and added CommentScore instead

* Uncovered enum case

* clippy

* reset translation changes

* translations

* translations

* Added additional hot ordering on controversial posts and comments

* featured local and featured community added to controversy rank index, additional order_by removed (?), added post_score and post_count to PersonSortType

* Woodpecker rerun

* cargo fmt

* woodpecker rerun

* fixed controversy_rank order

* fix

* Readded migration as latest, removed second update statement for setting controversy rank

crates/apub/src/api/search.rs

@@ -8,7 +8,7 @@ use lemmy_api_common::{
 };
 use lemmy_db_schema::{
   source::{community::Community, local_site::LocalSite},
-  utils::post_to_comment_sort_type,
+  utils::{post_to_comment_sort_type, post_to_person_sort_type},
   SearchType,
 };
 use lemmy_db_views::{comment_view::CommentQuery, post_view::PostQuery};
@@ -98,7 +98,7 @@ pub async fn search(
     }
     SearchType::Users => {
       users = PersonQuery {
-        sort: (sort),
+        sort: (sort.map(post_to_person_sort_type)),
         search_term: (Some(q)),
         page: (page),
         limit: (limit),
@@ -168,7 +168,7 @@ pub async fn search(
         vec![]
       } else {
         PersonQuery {
-          sort: (sort),
+          sort: (sort.map(post_to_person_sort_type)),
           search_term: (Some(q)),
           page: (page),
           limit: (limit),

crates/db_schema/src/aggregates/structs.rs

@@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize};
 #[cfg(feature = "full")]
 use ts_rs::TS;
 
-#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)]
+#[derive(PartialEq, Debug, Serialize, Deserialize, Clone)]
 #[cfg_attr(feature = "full", derive(Queryable, Associations, Identifiable, TS))]
 #[cfg_attr(feature = "full", diesel(table_name = comment_aggregates))]
 #[cfg_attr(feature = "full", diesel(belongs_to(crate::source::comment::Comment)))]
@@ -28,6 +28,7 @@ pub struct CommentAggregates {
   /// The total number of children in this comment branch.
   pub child_count: i32,
   pub hot_rank: i32,
+  pub controversy_rank: f64,
 }
 
 #[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)]
@@ -72,7 +73,7 @@ pub struct PersonAggregates {
   pub comment_score: i64,
 }
 
-#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)]
+#[derive(PartialEq, Debug, Serialize, Deserialize, Clone)]
 #[cfg_attr(feature = "full", derive(Queryable, Associations, Identifiable, TS))]
 #[cfg_attr(feature = "full", diesel(table_name = post_aggregates))]
 #[cfg_attr(feature = "full", diesel(belongs_to(crate::source::post::Post)))]
@@ -98,6 +99,7 @@ pub struct PostAggregates {
   pub hot_rank_active: i32,
   pub community_id: CommunityId,
   pub creator_id: PersonId,
+  pub controversy_rank: f64,
 }
 
 #[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)]

crates/db_schema/src/lib.rs

@@ -66,6 +66,7 @@ pub enum SortType {
   TopThreeMonths,
   TopSixMonths,
   TopNineMonths,
+  Controversial,
 }
 
 #[derive(EnumString, Display, Debug, Serialize, Deserialize, Clone, Copy)]
@@ -77,6 +78,20 @@ pub enum CommentSortType {
   Top,
   New,
   Old,
+  Controversial,
+}
+
+#[derive(EnumString, Display, Debug, Serialize, Deserialize, Clone, Copy)]
+#[cfg_attr(feature = "full", derive(TS))]
+#[cfg_attr(feature = "full", ts(export))]
+/// The person sort types. See here for descriptions: https://join-lemmy.org/docs/en/users/03-votes-and-ranking.html
+pub enum PersonSortType {
+  New,
+  Old,
+  MostComments,
+  CommentScore,
+  PostScore,
+  PostCount,
 }
 
 #[derive(EnumString, Display, Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)]

crates/db_schema/src/schema.rs

@@ -93,6 +93,7 @@ diesel::table! {
         published -> Timestamp,
         child_count -> Int4,
         hot_rank -> Int4,
+        controversy_rank -> Float8,
     }
 }
 
@@ -676,6 +677,7 @@ diesel::table! {
         hot_rank_active -> Int4,
         community_id -> Int4,
         creator_id -> Int4,
+        controversy_rank -> Float8,
     }
 }
 

crates/db_schema/src/utils.rs

@@ -3,6 +3,7 @@ use crate::{
   diesel_migrations::MigrationHarness,
   newtypes::DbUrl,
   CommentSortType,
+  PersonSortType,
   SortType,
 };
 use activitypub_federation::{fetch::object_id::ObjectId, traits::Object};
@@ -347,6 +348,7 @@ pub fn post_to_comment_sort_type(sort: SortType) -> CommentSortType {
     SortType::Active | SortType::Hot => CommentSortType::Hot,
     SortType::New | SortType::NewComments | SortType::MostComments => CommentSortType::New,
     SortType::Old => CommentSortType::Old,
+    SortType::Controversial => CommentSortType::Controversial,
     SortType::TopHour
     | SortType::TopSixHour
     | SortType::TopTwelveHour
@@ -361,6 +363,16 @@ pub fn post_to_comment_sort_type(sort: SortType) -> CommentSortType {
   }
 }
 
+pub fn post_to_person_sort_type(sort: SortType) -> PersonSortType {
+  match sort {
+    SortType::Active | SortType::Hot | SortType::Controversial => PersonSortType::CommentScore,
+    SortType::New | SortType::NewComments => PersonSortType::New,
+    SortType::MostComments => PersonSortType::MostComments,
+    SortType::Old => PersonSortType::Old,
+    _ => PersonSortType::CommentScore,
+  }
+}
+
 static EMAIL_REGEX: Lazy<Regex> = Lazy::new(|| {
   Regex::new(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$")
     .expect("compile email regex")
@@ -373,6 +385,10 @@ pub mod functions {
     fn hot_rank(score: BigInt, time: Timestamp) -> Integer;
   }
 
+  sql_function! {
+    fn controversy_rank(upvotes: BigInt, downvotes: BigInt, score: BigInt) -> Double;
+  }
+
   sql_function!(fn lower(x: Text) -> Text);
 }
 

crates/db_views/src/comment_report_view.rs

@@ -478,6 +478,7 @@ mod tests {
         published: agg.published,
         child_count: 0,
         hot_rank: 1728,
+        controversy_rank: 0.0,
       },
       my_vote: None,
       resolver: None,

crates/db_views/src/comment_view.rs

@@ -366,6 +366,9 @@ impl<'a> CommentQuery<'a> {
       CommentSortType::Hot => query
         .then_order_by(comment_aggregates::hot_rank.desc())
         .then_order_by(comment_aggregates::score.desc()),
+      CommentSortType::Controversial => {
+        query.then_order_by(comment_aggregates::controversy_rank.desc())
+      }
       CommentSortType::New => query.then_order_by(comment::published.desc()),
       CommentSortType::Old => query.then_order_by(comment::published.asc()),
       CommentSortType::Top => query.order_by(comment_aggregates::score.desc()),
@@ -948,6 +951,7 @@ mod tests {
         published: agg.published,
         child_count: 5,
         hot_rank: 1728,
+        controversy_rank: 0.0,
       },
     }
   }

crates/db_views/src/post_report_view.rs

@@ -470,6 +470,7 @@ mod tests {
         featured_local: false,
         hot_rank: 1728,
         hot_rank_active: 1728,
+        controversy_rank: 0.0,
         community_id: inserted_post.community_id,
         creator_id: inserted_post.creator_id,
       },

crates/db_views/src/post_view.rs

@@ -422,6 +422,7 @@ impl<'a> PostQuery<'a> {
       SortType::Hot => query
         .then_order_by(post_aggregates::hot_rank.desc())
         .then_order_by(post_aggregates::published.desc()),
+      SortType::Controversial => query.then_order_by(post_aggregates::controversy_rank.desc()),
       SortType::New => query.then_order_by(post_aggregates::published.desc()),
       SortType::Old => query.then_order_by(post_aggregates::published.asc()),
       SortType::NewComments => query.then_order_by(post_aggregates::newest_comment_time.desc()),
@@ -1141,6 +1142,7 @@ mod tests {
         featured_local: false,
         hot_rank: 1728,
         hot_rank_active: 1728,
+        controversy_rank: 0.0,
         community_id: inserted_post.community_id,
         creator_id: inserted_post.creator_id,
       },

crates/db_views/src/structs.rs

@@ -25,7 +25,7 @@ use serde_with::skip_serializing_none;
 use ts_rs::TS;
 
 #[skip_serializing_none]
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)]
+#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
 #[cfg_attr(feature = "full", derive(TS))]
 #[cfg_attr(feature = "full", ts(export))]
 /// A comment report view.
@@ -43,7 +43,7 @@ pub struct CommentReportView {
 }
 
 #[skip_serializing_none]
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)]
+#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
 #[cfg_attr(feature = "full", derive(TS))]
 #[cfg_attr(feature = "full", ts(export))]
 /// A comment view.
@@ -71,7 +71,7 @@ pub struct LocalUserView {
 }
 
 #[skip_serializing_none]
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)]
+#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
 #[cfg_attr(feature = "full", derive(TS))]
 #[cfg_attr(feature = "full", ts(export))]
 /// A post report view.
@@ -88,7 +88,7 @@ pub struct PostReportView {
 }
 
 #[skip_serializing_none]
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)]
+#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
 #[cfg_attr(feature = "full", derive(TS))]
 #[cfg_attr(feature = "full", ts(export))]
 /// A post view.

crates/db_views_actor/src/comment_reply_view.rs

@@ -266,6 +266,9 @@ impl CommentReplyQuery {
 
     query = match self.sort.unwrap_or(CommentSortType::New) {
       CommentSortType::Hot => query.then_order_by(comment_aggregates::hot_rank.desc()),
+      CommentSortType::Controversial => {
+        query.then_order_by(comment_aggregates::controversy_rank.desc())
+      }
       CommentSortType::New => query.then_order_by(comment_reply::published.desc()),
       CommentSortType::Old => query.then_order_by(comment_reply::published.asc()),
       CommentSortType::Top => query.order_by(comment_aggregates::score.desc()),

crates/db_views_actor/src/community_view.rs

@@ -170,7 +170,8 @@ impl<'a> CommunityQuery<'a> {
       }
       New => query = query.order_by(community::published.desc()),
       Old => query = query.order_by(community::published.asc()),
-      MostComments => query = query.order_by(community_aggregates::comments.desc()),
+      // Controversial is temporary until a CommentSortType is created
+      MostComments | Controversial => query = query.order_by(community_aggregates::comments.desc()),
       TopAll | TopYear | TopNineMonths => {
         query = query.order_by(community_aggregates::subscribers.desc())
       }

crates/db_views_actor/src/person_mention_view.rs

@@ -271,6 +271,9 @@ impl PersonMentionQuery {
 
     query = match self.sort.unwrap_or(CommentSortType::Hot) {
       CommentSortType::Hot => query.then_order_by(comment_aggregates::hot_rank.desc()),
+      CommentSortType::Controversial => {
+        query.then_order_by(comment_aggregates::controversy_rank.desc())
+      }
       CommentSortType::New => query.then_order_by(comment::published.desc()),
       CommentSortType::Old => query.then_order_by(comment::published.asc()),
       CommentSortType::Top => query.order_by(comment_aggregates::score.desc()),

crates/db_views_actor/src/person_view.rs

@@ -1,6 +1,6 @@
 use crate::structs::PersonView;
 use diesel::{
-  dsl::{now, IntervalDsl},
+  dsl::now,
   result::Error,
   BoolExpressionMethods,
   ExpressionMethods,
@@ -16,7 +16,7 @@ use lemmy_db_schema::{
   source::person::Person,
   traits::JoinView,
   utils::{fuzzy_search, get_conn, limit_and_offset, DbPool},
-  SortType,
+  PersonSortType,
 };
 use std::iter::Iterator;
 
@@ -80,7 +80,7 @@ impl PersonView {
 
 #[derive(Default)]
 pub struct PersonQuery {
-  pub sort: Option<SortType>,
+  pub sort: Option<PersonSortType>,
   pub search_term: Option<String>,
   pub page: Option<i64>,
   pub limit: Option<i64>,
@@ -101,43 +101,13 @@ impl PersonQuery {
         .or_filter(person::display_name.ilike(searcher));
     }
 
-    query = match self.sort.unwrap_or(SortType::Hot) {
+    query = match self.sort.unwrap_or(PersonSortType::CommentScore) {
-      SortType::New | SortType::NewComments => query.order_by(person::published.desc()),
+      PersonSortType::New => query.order_by(person::published.desc()),
-      SortType::Old => query.order_by(person::published.asc()),
+      PersonSortType::Old => query.order_by(person::published.asc()),
-      SortType::Hot | SortType::Active | SortType::TopAll => {
+      PersonSortType::MostComments => query.order_by(person_aggregates::comment_count.desc()),
-        query.order_by(person_aggregates::comment_score.desc())
+      PersonSortType::CommentScore => query.order_by(person_aggregates::comment_score.desc()),
-      }
+      PersonSortType::PostScore => query.order_by(person_aggregates::post_score.desc()),
-      SortType::MostComments => query.order_by(person_aggregates::comment_count.desc()),
+      PersonSortType::PostCount => query.order_by(person_aggregates::post_count.desc()),
-      SortType::TopYear => query
-        .filter(person::published.gt(now - 1.years()))
-        .order_by(person_aggregates::comment_score.desc()),
-      SortType::TopMonth => query
-        .filter(person::published.gt(now - 1.months()))
-        .order_by(person_aggregates::comment_score.desc()),
-      SortType::TopWeek => query
-        .filter(person::published.gt(now - 1.weeks()))
-        .order_by(person_aggregates::comment_score.desc()),
-      SortType::TopDay => query
-        .filter(person::published.gt(now - 1.days()))
-        .order_by(person_aggregates::comment_score.desc()),
-      SortType::TopHour => query
-        .filter(person::published.gt(now - 1.hours()))
-        .order_by(person_aggregates::comment_score.desc()),
-      SortType::TopSixHour => query
-        .filter(person::published.gt(now - 6.hours()))
-        .order_by(person_aggregates::comment_score.desc()),
-      SortType::TopTwelveHour => query
-        .filter(person::published.gt(now - 12.hours()))
-        .order_by(person_aggregates::comment_score.desc()),
-      SortType::TopThreeMonths => query
-        .filter(person::published.gt(now - 3.months()))
-        .order_by(person_aggregates::comment_score.desc()),
-      SortType::TopSixMonths => query
-        .filter(person::published.gt(now - 6.months()))
-        .order_by(person_aggregates::comment_score.desc()),
-      SortType::TopNineMonths => query
-        .filter(person::published.gt(now - 9.months()))
-        .order_by(person_aggregates::comment_score.desc()),
     };
 
     let (limit, offset) = limit_and_offset(self.page, self.limit)?;

crates/db_views_actor/src/structs.rs

@@ -70,7 +70,7 @@ pub struct PersonBlockView {
 }
 
 #[skip_serializing_none]
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)]
+#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
 #[cfg_attr(feature = "full", derive(TS))]
 #[cfg_attr(feature = "full", ts(export))]
 /// A person mention view.
@@ -90,7 +90,7 @@ pub struct PersonMentionView {
 }
 
 #[skip_serializing_none]
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)]
+#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
 #[cfg_attr(feature = "full", derive(TS))]
 #[cfg_attr(feature = "full", ts(export))]
 /// A comment reply view.

migrations/2023-07-26-000217_create_controversial_indexes/down.sql

@@ -0,0 +1,63 @@
+-- Update comment_aggregates_score trigger function to exclude controversy_rank update
+create or replace function comment_aggregates_score()
+returns trigger language plpgsql
+as $$
+begin
+  IF (TG_OP = 'INSERT') THEN
+    update comment_aggregates ca
+    set score = score + NEW.score,
+    upvotes = case when NEW.score = 1 then upvotes + 1 else upvotes end,
+    downvotes = case when NEW.score = -1 then downvotes + 1 else downvotes end
+    where ca.comment_id = NEW.comment_id;
+
+  ELSIF (TG_OP = 'DELETE') THEN
+    -- Join to comment because that comment may not exist anymore
+    update comment_aggregates ca
+    set score = score - OLD.score,
+    upvotes = case when OLD.score = 1 then upvotes - 1 else upvotes end,
+    downvotes = case when OLD.score = -1 then downvotes - 1 else downvotes end
+    from comment c
+    where ca.comment_id = c.id
+    and ca.comment_id = OLD.comment_id;
+
+  END IF;
+  return null;
+end $$;
+
+-- Update post_aggregates_score trigger function to exclude controversy_rank update
+create or replace function post_aggregates_score()
+returns trigger language plpgsql
+as $$
+begin
+  IF (TG_OP = 'INSERT') THEN
+    update post_aggregates pa
+    set score = score + NEW.score,
+    upvotes = case when NEW.score = 1 then upvotes + 1 else upvotes end,
+    downvotes = case when NEW.score = -1 then downvotes + 1 else downvotes end
+    where pa.post_id = NEW.post_id;
+
+  ELSIF (TG_OP = 'DELETE') THEN
+    -- Join to post because that post may not exist anymore
+    update post_aggregates pa
+    set score = score - OLD.score,
+    upvotes = case when OLD.score = 1 then upvotes - 1 else upvotes end,
+    downvotes = case when OLD.score = -1 then downvotes - 1 else downvotes end
+    from post p
+    where pa.post_id = p.id
+    and pa.post_id = OLD.post_id;
+  END IF;
+  return null;
+end $$;
+
+-- Drop the indexes
+drop index if exists idx_post_aggregates_featured_local_controversy;
+drop index if exists idx_post_aggregates_featured_community_controversy;
+drop index if exists idx_comment_aggregates_controversy;
+
+-- Remove the added columns from the tables
+alter table post_aggregates drop column controversy_rank;
+alter table comment_aggregates drop column controversy_rank;
+
+-- Remove function
+drop function controversy_rank(numeric, numeric);
+

migrations/2023-07-26-000217_create_controversial_indexes/up.sql

@@ -0,0 +1,97 @@
+-- Need to add immutable to the controversy_rank function in order to index by it
+
+-- Controversy Rank:
+--      if downvotes <= 0 or upvotes <= 0:
+--          0 
+--      else:
+--          (upvotes + downvotes) * min(upvotes, downvotes) / max(upvotes, downvotes) 
+create or replace function controversy_rank(upvotes numeric, downvotes numeric)
+returns float as $$
+begin
+    if downvotes <= 0 or upvotes <= 0 then
+        return 0;
+    else
+        return (upvotes + downvotes) *
+            case when upvotes > downvotes
+                then downvotes::float / upvotes::float
+                else upvotes::float / downvotes::float
+            end;
+    end if;
+end; $$
+LANGUAGE plpgsql
+IMMUTABLE;
+
+-- Aggregates
+alter table post_aggregates add column controversy_rank float not null default 0;
+alter table comment_aggregates add column controversy_rank float not null default 0;
+
+-- Populate them initially
+-- Note: After initial population, these are updated with vote triggers
+update post_aggregates set controversy_rank = controversy_rank(upvotes::numeric, downvotes::numeric);
+update comment_aggregates set controversy_rank = controversy_rank(upvotes::numeric, downvotes::numeric);
+
+-- Create single column indexes
+create index idx_post_aggregates_featured_local_controversy on post_aggregates (featured_local desc, controversy_rank desc);
+create index idx_post_aggregates_featured_community_controversy on post_aggregates (featured_community desc, controversy_rank desc);
+create index idx_comment_aggregates_controversy on comment_aggregates (controversy_rank desc);
+
+-- Update post_aggregates_score trigger function to include controversy_rank update
+create or replace function post_aggregates_score()
+returns trigger language plpgsql
+as $$
+begin
+  IF (TG_OP = 'INSERT') THEN
+    update post_aggregates pa
+    set score = score + NEW.score,
+    upvotes = case when NEW.score = 1 then upvotes + 1 else upvotes end,
+    downvotes = case when NEW.score = -1 then downvotes + 1 else downvotes end,
+    controversy_rank = controversy_rank(pa.upvotes + case when NEW.score = 1 then 1 else 0 end::numeric, 
+                                         pa.downvotes + case when NEW.score = -1 then 1 else 0 end::numeric)
+    where pa.post_id = NEW.post_id;
+
+  ELSIF (TG_OP = 'DELETE') THEN
+    -- Join to post because that post may not exist anymore
+    update post_aggregates pa
+    set score = score - OLD.score,
+    upvotes = case when OLD.score = 1 then upvotes - 1 else upvotes end,
+    downvotes = case when OLD.score = -1 then downvotes - 1 else downvotes end,
+    controversy_rank = controversy_rank(pa.upvotes + case when NEW.score = 1 then 1 else 0 end::numeric, 
+                                         pa.downvotes + case when NEW.score = -1 then 1 else 0 end::numeric)
+    from post p
+    where pa.post_id = p.id
+    and pa.post_id = OLD.post_id;
+
+  END IF;
+  return null;
+end $$;
+
+-- Update comment_aggregates_score trigger function to include controversy_rank update
+create or replace function comment_aggregates_score()
+returns trigger language plpgsql
+as $$
+begin
+  IF (TG_OP = 'INSERT') THEN
+    update comment_aggregates ca
+    set score = score + NEW.score,
+    upvotes = case when NEW.score = 1 then upvotes + 1 else upvotes end,
+    downvotes = case when NEW.score = -1 then downvotes + 1 else downvotes end,
+    controversy_rank = controversy_rank(ca.upvotes + case when NEW.score = 1 then 1 else 0 end::numeric, 
+                                         ca.downvotes + case when NEW.score = -1 then 1 else 0 end::numeric)
+    where ca.comment_id = NEW.comment_id;
+
+  ELSIF (TG_OP = 'DELETE') THEN
+    -- Join to comment because that comment may not exist anymore
+    update comment_aggregates ca
+    set score = score - OLD.score,
+    upvotes = case when OLD.score = 1 then upvotes - 1 else upvotes end,
+    downvotes = case when OLD.score = -1 then downvotes - 1 else downvotes end,
+    controversy_rank = controversy_rank(ca.upvotes + case when NEW.score = 1 then 1 else 0 end::numeric, 
+                                         ca.downvotes + case when NEW.score = -1 then 1 else 0 end::numeric)
+    from comment c
+    where ca.comment_id = c.id
+    and ca.comment_id = OLD.comment_id;
+
+  END IF;
+  return null;
+end $$;
+