grishka/lemmy
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Fix Posts List Performance + cursor-based pagination (#3872)

Browse source 
* add token-based pagination + fast subscribed post view

* add migrations

* fix failing heuristic

* revert

* output pagination token as next_page, fix off-by-one, restructure

* more cleanup

* clean

* format sql

* fix comment

* fix tests

* e

* empty

* move last page thing

* restructure a bit for readability

* rename page_cursor

* update for scaled sort

* fix

* sql format

* fix

* get rid of macros

---------

Co-authored-by: Dessalines <dessalines@users.noreply.github.com>
This commit is contained in:
phiresky 2023-09-18 15:44:48 +02:00 committed by GitHub
parent 6735a98d35
commit 544d30f0d4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 414 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
crates/api_common/src/post.rs
View file

@ -5,7 +5,7 @@ use lemmy_db_schema::{
PostFeatureType, PostFeatureType,
SortType, SortType,
}; };
use lemmy_db_views::structs::{PostReportView, PostView}; use lemmy_db_views::structs::{PaginationCursor, PostReportView, PostView};
use lemmy_db_views_actor::structs::{CommunityModeratorView, CommunityView}; use lemmy_db_views_actor::structs::{CommunityModeratorView, CommunityView};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_with::skip_serializing_none; use serde_with::skip_serializing_none;
@ -70,6 +70,7 @@ pub struct GetPostResponse {
pub struct GetPosts { pub struct GetPosts {
pub type_: Option<ListingType>, pub type_: Option<ListingType>,
pub sort: Option<SortType>, pub sort: Option<SortType>,
/// DEPRECATED, use page_cursor
pub page: Option<i64>, pub page: Option<i64>,
pub limit: Option<i64>, pub limit: Option<i64>,
pub community_id: Option<CommunityId>, pub community_id: Option<CommunityId>,
@ -78,6 +79,7 @@ pub struct GetPosts {
pub liked_only: Option<bool>, pub liked_only: Option<bool>,
pub disliked_only: Option<bool>, pub disliked_only: Option<bool>,
pub auth: Option<Sensitive<String>>, pub auth: Option<Sensitive<String>>,
pub page_cursor: Option<PaginationCursor>,
} }
#[derive(Serialize, Deserialize, Debug, Clone)] #[derive(Serialize, Deserialize, Debug, Clone)]
@ -86,6 +88,8 @@ pub struct GetPosts {
/// The post list response. /// The post list response.
pub struct GetPostsResponse { pub struct GetPostsResponse {
pub posts: Vec<PostView>, pub posts: Vec<PostView>,
/// the pagination cursor to use to fetch the next page
pub next_page: Option<PaginationCursor>,
} }
#[derive(Debug, Serialize, Deserialize, Clone, Default)] #[derive(Debug, Serialize, Deserialize, Clone, Default)]

16
crates/apub/src/api/list_posts.rs
View file

@ -11,7 +11,10 @@ use lemmy_api_common::{
utils::{check_private_instance, local_user_view_from_jwt_opt_new}, utils::{check_private_instance, local_user_view_from_jwt_opt_new},
}; };
use lemmy_db_schema::source::{community::Community, local_site::LocalSite}; use lemmy_db_schema::source::{community::Community, local_site::LocalSite};
use lemmy_db_views::{post_view::PostQuery, structs::LocalUserView}; use lemmy_db_views::{
post_view::PostQuery,
structs::{LocalUserView, PaginationCursor},
};
use lemmy_utils::error::{LemmyError, LemmyErrorExt, LemmyErrorType}; use lemmy_utils::error::{LemmyError, LemmyErrorExt, LemmyErrorType};
#[tracing::instrument(skip(context))] #[tracing::instrument(skip(context))]
@ -48,6 +51,12 @@ pub async fn list_posts(
&local_site, &local_site,
community_id, community_id,
)?); )?);
// parse pagination token
let page_after = if let Some(pa) = &data.page_cursor {
Some(pa.read(&mut context.pool()).await?)
} else {
None
};
let posts = PostQuery { let posts = PostQuery {
local_user: local_user_view.as_ref(), local_user: local_user_view.as_ref(),
@ -58,6 +67,7 @@ pub async fn list_posts(
liked_only, liked_only,
disliked_only, disliked_only,
page, page,
page_after,
limit, limit,
..Default::default() ..Default::default()
} }
@ -65,5 +75,7 @@ pub async fn list_posts(
.await .await
.with_lemmy_type(LemmyErrorType::CouldntGetPosts)?; .with_lemmy_type(LemmyErrorType::CouldntGetPosts)?;
Ok(Json(GetPostsResponse { posts })) // if this page wasn't empty, then there is a next page after the last post on this page
let next_page = posts.last().map(PaginationCursor::after_post);
Ok(Json(GetPostsResponse { posts, next_page }))
} }

352
crates/db_views/src/post_view.rs
View file

@ -1,21 +1,26 @@
use crate::structs::{LocalUserView, PostView}; use crate::structs::{LocalUserView, PaginationCursor, PostView};
use diesel::{ use diesel::{
debug_query, debug_query,
dsl::{exists, not, IntervalDsl}, dsl::{self, exists, not, IntervalDsl},
expression::AsExpression,
pg::Pg, pg::Pg,
result::Error, result::Error,
sql_function, sql_function,
sql_types::{self, Timestamptz}, sql_types::{self, SingleValue, SqlType, Timestamptz},
BoolExpressionMethods, BoolExpressionMethods,
BoxableExpression, BoxableExpression,
Expression,
ExpressionMethods, ExpressionMethods,
IntoSql, IntoSql,
JoinOnDsl,
NullableExpressionMethods, NullableExpressionMethods,
OptionalExtension,
PgTextExpressionMethods, PgTextExpressionMethods,
QueryDsl, QueryDsl,
}; };
use diesel_async::RunQueryDsl; use diesel_async::RunQueryDsl;
use lemmy_db_schema::{ use lemmy_db_schema::{
aggregates::structs::PostAggregates,
newtypes::{CommunityId, LocalUserId, PersonId, PostId}, newtypes::{CommunityId, LocalUserId, PersonId, PostId},
schema::{ schema::{
community, community,
@ -28,12 +33,12 @@ use lemmy_db_schema::{
person_block, person_block,
person_post_aggregates, person_post_aggregates,
post, post,
post_aggregates, post_aggregates::{self, newest_comment_time},
post_like, post_like,
post_read, post_read,
post_saved, post_saved,
}, },
utils::{fuzzy_search, limit_and_offset, DbConn, DbPool, ListFn, Queries, ReadFn}, utils::{fuzzy_search, get_conn, limit_and_offset, DbConn, DbPool, ListFn, Queries, ReadFn},
ListingType, ListingType,
SortType, SortType,
}; };
@ -41,6 +46,55 @@ use tracing::debug;
sql_function!(fn coalesce(x: sql_types::Nullable<sql_types::BigInt>, y: sql_types::BigInt) -> sql_types::BigInt); sql_function!(fn coalesce(x: sql_types::Nullable<sql_types::BigInt>, y: sql_types::BigInt) -> sql_types::BigInt);
fn order_and_page_filter_desc<Q, C, T>(
query: Q,
column: C,
options: &PostQuery,
getter: impl Fn(&PostAggregates) -> T,
) -> Q
where
Q: diesel::query_dsl::methods::ThenOrderDsl<dsl::Desc<C>, Output = Q>
+ diesel::query_dsl::methods::ThenOrderDsl<dsl::Asc<C>, Output = Q>
+ diesel::query_dsl::methods::FilterDsl<dsl::GtEq<C, T>, Output = Q>
+ diesel::query_dsl::methods::FilterDsl<dsl::LtEq<C, T>, Output = Q>,
C: Expression + Copy,
C::SqlType: SingleValue + SqlType,
T: AsExpression<C::SqlType>,
{
let mut query = query.then_order_by(column.desc());
if let Some(before) = &options.page_before_or_equal {
query = query.filter(column.ge(getter(&before.0)));
}
if let Some(after) = &options.page_after {
query = query.filter(column.le(getter(&after.0)));
}
query
}
fn order_and_page_filter_asc<Q, C, T>(
query: Q,
column: C,
options: &PostQuery,
getter: impl Fn(&PostAggregates) -> T,
) -> Q
where
Q: diesel::query_dsl::methods::ThenOrderDsl<dsl::Asc<C>, Output = Q>
+ diesel::query_dsl::methods::FilterDsl<dsl::LtEq<C, T>, Output = Q>
+ diesel::query_dsl::methods::FilterDsl<dsl::GtEq<C, T>, Output = Q>,
C: Expression + Copy,
C::SqlType: SingleValue + SqlType,
T: AsExpression<C::SqlType>,
{
let mut query = query.then_order_by(column.asc());
if let Some(before) = &options.page_before_or_equal {
query = query.filter(column.le(getter(&before.0)));
}
if let Some(after) = &options.page_after {
query = query.filter(column.ge(getter(&after.0)));
}
query
}
fn queries<'a>() -> Queries< fn queries<'a>() -> Queries<
impl ReadFn<'a, PostView, (PostId, Option<PersonId>, bool)>, impl ReadFn<'a, PostView, (PostId, Option<PersonId>, bool)>,
impl ListFn<'a, PostView, PostQuery<'a>>, impl ListFn<'a, PostView, PostQuery<'a>>,
@ -251,13 +305,18 @@ fn queries<'a>() -> Queries<
.filter(community::removed.eq(false)) .filter(community::removed.eq(false))
.filter(post::removed.eq(false)); .filter(post::removed.eq(false));
} }
if options.community_id.is_none() || options.community_id_just_for_prefetch {
if options.community_id.is_none() { query = order_and_page_filter_desc(query, post_aggregates::featured_local, &options, |e| {
query = query.then_order_by(post_aggregates::featured_local.desc()); e.featured_local
} else if let Some(community_id) = options.community_id { });
query = query } else {
.filter(post_aggregates::community_id.eq(community_id)) query =
.then_order_by(post_aggregates::featured_community.desc()); order_and_page_filter_desc(query, post_aggregates::featured_community, &options, |e| {
e.featured_community
});
}
if let Some(community_id) = options.community_id {
query = query.filter(post_aggregates::community_id.eq(community_id));
} }
if let Some(creator_id) = options.creator_id { if let Some(creator_id) = options.creator_id {
@ -292,12 +351,12 @@ fn queries<'a>() -> Queries<
} }
} }
if let Some(url_search) = options.url_search { if let Some(url_search) = &options.url_search {
query = query.filter(post::url.eq(url_search)); query = query.filter(post::url.eq(url_search));
} }
if let Some(search_term) = options.search_term { if let Some(search_term) = &options.search_term {
let searcher = fuzzy_search(&search_term); let searcher = fuzzy_search(search_term);
query = query.filter( query = query.filter(
post::name post::name
.ilike(searcher.clone()) .ilike(searcher.clone())
@ -373,70 +432,91 @@ fn queries<'a>() -> Queries<
} }
let now = diesel::dsl::now.into_sql::<Timestamptz>(); let now = diesel::dsl::now.into_sql::<Timestamptz>();
query = match options.sort.unwrap_or(SortType::Hot) { {
SortType::Active => query use post_aggregates::{
.then_order_by(post_aggregates::hot_rank_active.desc()) comments,
.then_order_by(post_aggregates::published.desc()), controversy_rank,
SortType::Hot => query hot_rank,
.then_order_by(post_aggregates::hot_rank.desc()) hot_rank_active,
.then_order_by(post_aggregates::published.desc()), published,
SortType::Scaled => query scaled_rank,
.then_order_by(post_aggregates::scaled_rank.desc()) score,
.then_order_by(post_aggregates::published.desc()), };
SortType::Controversial => query.then_order_by(post_aggregates::controversy_rank.desc()), match options.sort.as_ref().unwrap_or(&SortType::Hot) {
SortType::New => query.then_order_by(post_aggregates::published.desc()), SortType::Active => {
SortType::Old => query.then_order_by(post_aggregates::published.asc()), query =
SortType::NewComments => query.then_order_by(post_aggregates::newest_comment_time.desc()), order_and_page_filter_desc(query, hot_rank_active, &options, |e| e.hot_rank_active);
SortType::MostComments => query query = order_and_page_filter_desc(query, published, &options, |e| e.published);
.then_order_by(post_aggregates::comments.desc()) }
.then_order_by(post_aggregates::published.desc()), SortType::Hot => {
SortType::TopAll => query query = order_and_page_filter_desc(query, hot_rank, &options, |e| e.hot_rank);
.then_order_by(post_aggregates::score.desc()) query = order_and_page_filter_desc(query, published, &options, |e| e.published);
.then_order_by(post_aggregates::published.desc()), }
SortType::TopYear => query SortType::Scaled => {
.filter(post_aggregates::published.gt(now - 1.years())) query = order_and_page_filter_desc(query, scaled_rank, &options, |e| e.scaled_rank);
.then_order_by(post_aggregates::score.desc()) query = order_and_page_filter_desc(query, published, &options, |e| e.published);
.then_order_by(post_aggregates::published.desc()), }
SortType::TopMonth => query SortType::Controversial => {
.filter(post_aggregates::published.gt(now - 1.months())) query =
.then_order_by(post_aggregates::score.desc()) order_and_page_filter_desc(query, controversy_rank, &options, |e| e.controversy_rank);
.then_order_by(post_aggregates::published.desc()), query = order_and_page_filter_desc(query, published, &options, |e| e.published);
SortType::TopWeek => query }
.filter(post_aggregates::published.gt(now - 1.weeks())) SortType::New => {
.then_order_by(post_aggregates::score.desc()) query = order_and_page_filter_desc(query, published, &options, |e| e.published)
.then_order_by(post_aggregates::published.desc()), }
SortType::TopDay => query SortType::Old => {
.filter(post_aggregates::published.gt(now - 1.days())) query = order_and_page_filter_asc(query, published, &options, |e| e.published)
.then_order_by(post_aggregates::score.desc()) }
.then_order_by(post_aggregates::published.desc()), SortType::NewComments => {
SortType::TopHour => query query = order_and_page_filter_desc(query, newest_comment_time, &options, |e| {
.filter(post_aggregates::published.gt(now - 1.hours())) e.newest_comment_time
.then_order_by(post_aggregates::score.desc()) })
.then_order_by(post_aggregates::published.desc()), }
SortType::TopSixHour => query SortType::MostComments => {
.filter(post_aggregates::published.gt(now - 6.hours())) query = order_and_page_filter_desc(query, comments, &options, |e| e.comments);
.then_order_by(post_aggregates::score.desc()) query = order_and_page_filter_desc(query, published, &options, |e| e.published);
.then_order_by(post_aggregates::published.desc()), }
SortType::TopTwelveHour => query SortType::TopAll => {
.filter(post_aggregates::published.gt(now - 12.hours())) query = order_and_page_filter_desc(query, score, &options, |e| e.score);
.then_order_by(post_aggregates::score.desc()) query = order_and_page_filter_desc(query, published, &options, |e| e.published);
.then_order_by(post_aggregates::published.desc()), }
SortType::TopThreeMonths => query o @ (SortType::TopYear
.filter(post_aggregates::published.gt(now - 3.months())) | SortType::TopMonth
.then_order_by(post_aggregates::score.desc()) | SortType::TopWeek
.then_order_by(post_aggregates::published.desc()), | SortType::TopDay
SortType::TopSixMonths => query | SortType::TopHour
.filter(post_aggregates::published.gt(now - 6.months())) | SortType::TopSixHour
.then_order_by(post_aggregates::score.desc()) | SortType::TopTwelveHour
.then_order_by(post_aggregates::published.desc()), | SortType::TopThreeMonths
SortType::TopNineMonths => query | SortType::TopSixMonths
.filter(post_aggregates::published.gt(now - 9.months())) | SortType::TopNineMonths) => {
.then_order_by(post_aggregates::score.desc()) let interval = match o {
.then_order_by(post_aggregates::published.desc()), SortType::TopYear => 1.years(),
SortType::TopMonth => 1.months(),
SortType::TopWeek => 1.weeks(),
SortType::TopDay => 1.days(),
SortType::TopHour => 1.hours(),
SortType::TopSixHour => 6.hours(),
SortType::TopTwelveHour => 12.hours(),
SortType::TopThreeMonths => 3.months(),
SortType::TopSixMonths => 6.months(),
SortType::TopNineMonths => 9.months(),
_ => return Err(Error::NotFound),
};
query = query.filter(post_aggregates::published.gt(now - interval));
query = order_and_page_filter_desc(query, score, &options, |e| e.score);
query = order_and_page_filter_desc(query, published, &options, |e| e.published);
}
}
}; };
let (limit, offset) = limit_and_offset(options.page, options.limit)?; let (limit, mut offset) = limit_and_offset(options.page, options.limit)?;
if options.page_after.is_some() {
// always skip exactly one post because that's the last post of the previous page
// fixing the where clause is more difficult because we'd have to change only the last order-by-where clause
// e.g. WHERE (featured_local<=, hot_rank<=, published<=) to WHERE (<=, <=, <)
offset = 1;
}
query = query.limit(limit).offset(offset); query = query.limit(limit).offset(offset);
debug!("Post View Query: {:?}", debug_query::<Pg, _>(&query)); debug!("Post View Query: {:?}", debug_query::<Pg, _>(&query));
@ -468,12 +548,42 @@ impl PostView {
} }
} }
#[derive(Default)] impl PaginationCursor {
// get cursor for page that starts immediately after the given post
pub fn after_post(view: &PostView) -> PaginationCursor {
// hex encoding to prevent ossification
PaginationCursor(format!("P{:x}", view.counts.post_id.0))
}
pub async fn read(&self, pool: &mut DbPool<'_>) -> Result<PaginationCursorData, Error> {
Ok(PaginationCursorData(
PostAggregates::read(
pool,
PostId(
self
.0
.get(1..)
.and_then(|e| i32::from_str_radix(e, 16).ok())
.ok_or_else(|| Error::QueryBuilderError("Could not parse pagination token".into()))?,
),
)
.await?,
))
}
}
// currently we use a postaggregates struct as the pagination token.
// we only use some of the properties of the post aggregates, depending on which sort type we page by
#[derive(Clone)]
pub struct PaginationCursorData(PostAggregates);
#[derive(Default, Clone)]
pub struct PostQuery<'a> { pub struct PostQuery<'a> {
pub listing_type: Option<ListingType>, pub listing_type: Option<ListingType>,
pub sort: Option<SortType>, pub sort: Option<SortType>,
pub creator_id: Option<PersonId>, pub creator_id: Option<PersonId>,
pub community_id: Option<CommunityId>, pub community_id: Option<CommunityId>,
// if true, the query should be handled as if community_id was not given except adding the literal filter
pub community_id_just_for_prefetch: bool,
pub local_user: Option<&'a LocalUserView>, pub local_user: Option<&'a LocalUserView>,
pub search_term: Option<String>, pub search_term: Option<String>,
pub url_search: Option<String>, pub url_search: Option<String>,
@ -484,13 +594,99 @@ pub struct PostQuery<'a> {
pub is_profile_view: bool, pub is_profile_view: bool,
pub page: Option<i64>, pub page: Option<i64>,
pub limit: Option<i64>, pub limit: Option<i64>,
pub page_after: Option<PaginationCursorData>,
pub page_before_or_equal: Option<PaginationCursorData>,
} }
impl<'a> PostQuery<'a> { impl<'a> PostQuery<'a> {
async fn prefetch_upper_bound_for_page_before(
&self,
pool: &mut DbPool<'_>,
) -> Result<Option<PostQuery<'a>>, Error> {
// first get one page for the most popular community to get an upper bound for the the page end for the real query
// the reason this is needed is that when fetching posts for a single community PostgreSQL can optimize
// the query to use an index on e.g. (=, >=, >=, >=) and fetch only LIMIT rows
// but for the followed-communities query it has to query the index on (IN, >=, >=, >=)
// which it currently can't do at all (as of PG 16). see the discussion here:
// https://github.com/LemmyNet/lemmy/issues/2877#issuecomment-1673597190
//
// the results are correct no matter which community we fetch these for, since it basically covers the "worst case" of the whole page consisting of posts from one community
// but using the largest community decreases the pagination-frame so make the real query more efficient.
use lemmy_db_schema::schema::{
community_aggregates::dsl::{community_aggregates, community_id, users_active_month},
community_follower::dsl::{
community_follower,
community_id as follower_community_id,
person_id,
},
};
let (limit, offset) = limit_and_offset(self.page, self.limit)?;
if offset != 0 {
return Err(Error::QueryBuilderError(
"legacy pagination cannot be combined with v2 pagination".into(),
));
}
let self_person_id = self
.local_user
.expect("part of the above if")
.local_user
.person_id;
let largest_subscribed = {
let conn = &mut get_conn(pool).await?;
community_follower
.filter(person_id.eq(self_person_id))
.inner_join(community_aggregates.on(community_id.eq(follower_community_id)))
.order_by(users_active_month.desc())
.select(community_id)
.limit(1)
.get_result::<CommunityId>(conn)
.await
.optional()?
};
let Some(largest_subscribed) = largest_subscribed else {
// nothing subscribed to? no posts
return Ok(None);
};
let mut v = queries()
.list(
pool,
PostQuery {
community_id: Some(largest_subscribed),
community_id_just_for_prefetch: true,
..self.clone()
},
)
.await?;
// take last element of array. if this query returned less than LIMIT elements,
// the heuristic is invalid since we can't guarantee the full query will return >= LIMIT results (return original query)
if (v.len() as i64) < limit {
Ok(Some(self.clone()))
} else {
let page_before_or_equal = Some(PaginationCursorData(v.pop().expect("else case").counts));
Ok(Some(PostQuery {
page_before_or_equal,
..self.clone()
}))
}
}
pub async fn list(self, pool: &mut DbPool<'_>) -> Result<Vec<PostView>, Error> { pub async fn list(self, pool: &mut DbPool<'_>) -> Result<Vec<PostView>, Error> {
if self.listing_type == Some(ListingType::Subscribed)
&& self.community_id.is_none()
&& self.local_user.is_some()
&& self.page_before_or_equal.is_none()
{
if let Some(query) = self.prefetch_upper_bound_for_page_before(pool).await? {
queries().list(pool, query).await
} else {
Ok(vec![])
}
} else {
queries().list(pool, self).await queries().list(pool, self).await
} }
} }
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {

7
crates/db_views/src/structs.rs
View file

@ -89,6 +89,13 @@ pub struct PostReportView {
pub resolver: Option<Person>, pub resolver: Option<Person>,
} }
/// currently this is just a wrapper around post id, but should be seen as opaque from the client's perspective
/// stringified since we might want to use arbitrary info later, with a P prepended to prevent ossification
/// (api users love to make assumptions (e.g. parse stuff that looks like numbers as numbers) about apis that aren't part of the spec
#[derive(Serialize, Deserialize, Debug, Clone)]
#[cfg_attr(feature = "full", derive(ts_rs::TS))]
pub struct PaginationCursor(pub(crate) String);
#[skip_serializing_none] #[skip_serializing_none]
#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] #[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
#[cfg_attr(feature = "full", derive(TS, Queryable))] #[cfg_attr(feature = "full", derive(TS, Queryable))]

54
migrations/2023-09-07-215546_post-queries-efficient/down.sql Normal file
View file

@ -0,0 +1,54 @@
DROP INDEX idx_post_aggregates_featured_community_active;
DROP INDEX idx_post_aggregates_featured_community_controversy;
DROP INDEX idx_post_aggregates_featured_community_hot;
DROP INDEX idx_post_aggregates_featured_community_scaled;
DROP INDEX idx_post_aggregates_featured_community_most_comments;
DROP INDEX idx_post_aggregates_featured_community_newest_comment_time;
DROP INDEX idx_post_aggregates_featured_community_newest_comment_time_necro;
DROP INDEX idx_post_aggregates_featured_community_published;
DROP INDEX idx_post_aggregates_featured_community_score;
CREATE INDEX idx_post_aggregates_featured_community_active ON post_aggregates (featured_community DESC, hot_rank_active DESC, published DESC);
CREATE INDEX idx_post_aggregates_featured_community_controversy ON post_aggregates (featured_community DESC, controversy_rank DESC);
CREATE INDEX idx_post_aggregates_featured_community_hot ON post_aggregates (featured_community DESC, hot_rank DESC, published DESC);
CREATE INDEX idx_post_aggregates_featured_community_scaled ON post_aggregates (featured_community DESC, scaled_rank DESC, published DESC);
CREATE INDEX idx_post_aggregates_featured_community_most_comments ON post_aggregates (featured_community DESC, comments DESC, published DESC);
CREATE INDEX idx_post_aggregates_featured_community_newest_comment_time ON post_aggregates (featured_community DESC, newest_comment_time DESC);
CREATE INDEX idx_post_aggregates_featured_community_newest_comment_time_necro ON post_aggregates (featured_community DESC, newest_comment_time_necro DESC);
CREATE INDEX idx_post_aggregates_featured_community_published ON post_aggregates (featured_community DESC, published DESC);
CREATE INDEX idx_post_aggregates_featured_community_score ON post_aggregates (featured_community DESC, score DESC, published DESC);
DROP INDEX idx_post_aggregates_community_active;
DROP INDEX idx_post_aggregates_community_controversy;
DROP INDEX idx_post_aggregates_community_hot;
DROP INDEX idx_post_aggregates_community_scaled;
DROP INDEX idx_post_aggregates_community_most_comments;
DROP INDEX idx_post_aggregates_community_newest_comment_time;
DROP INDEX idx_post_aggregates_community_newest_comment_time_necro;
DROP INDEX idx_post_aggregates_community_published;
DROP INDEX idx_post_aggregates_community_score;

59
migrations/2023-09-07-215546_post-queries-efficient/up.sql Normal file
View file

@ -0,0 +1,59 @@
-- these indices are used for single-community filtering and for the followed-communities (front-page) query
-- basically one index per Sort
-- index name is truncated to 63 chars so abbreviate a bit
CREATE INDEX idx_post_aggregates_community_active ON post_aggregates (community_id, featured_local DESC, hot_rank_active DESC, published DESC);
CREATE INDEX idx_post_aggregates_community_controversy ON post_aggregates (community_id, featured_local DESC, controversy_rank DESC);
CREATE INDEX idx_post_aggregates_community_hot ON post_aggregates (community_id, featured_local DESC, hot_rank DESC, published DESC);
CREATE INDEX idx_post_aggregates_community_scaled ON post_aggregates (community_id, featured_local DESC, scaled_rank DESC, published DESC);
CREATE INDEX idx_post_aggregates_community_most_comments ON post_aggregates (community_id, featured_local DESC, comments DESC, published DESC);
CREATE INDEX idx_post_aggregates_community_newest_comment_time ON post_aggregates (community_id, featured_local DESC, newest_comment_time DESC);
CREATE INDEX idx_post_aggregates_community_newest_comment_time_necro ON post_aggregates (community_id, featured_local DESC, newest_comment_time_necro DESC);
CREATE INDEX idx_post_aggregates_community_published ON post_aggregates (community_id, featured_local DESC, published DESC);
CREATE INDEX idx_post_aggregates_community_score ON post_aggregates (community_id, featured_local DESC, score DESC, published DESC);
-- these indices are used for "per-community" filtering
-- these indices weren't really useful because whenever the query filters by featured_community it also filters by community_id, so prepend that to all these indexes
DROP INDEX idx_post_aggregates_featured_community_active;
DROP INDEX idx_post_aggregates_featured_community_controversy;
DROP INDEX idx_post_aggregates_featured_community_hot;
DROP INDEX idx_post_aggregates_featured_community_scaled;
DROP INDEX idx_post_aggregates_featured_community_most_comments;
DROP INDEX idx_post_aggregates_featured_community_newest_comment_time;
DROP INDEX idx_post_aggregates_featured_community_newest_comment_time_necro;
DROP INDEX idx_post_aggregates_featured_community_published;
DROP INDEX idx_post_aggregates_featured_community_score;
CREATE INDEX idx_post_aggregates_featured_community_active ON post_aggregates (community_id, featured_community DESC, hot_rank_active DESC, published DESC);
CREATE INDEX idx_post_aggregates_featured_community_controversy ON post_aggregates (community_id, featured_community DESC, controversy_rank DESC);
CREATE INDEX idx_post_aggregates_featured_community_hot ON post_aggregates (community_id, featured_community DESC, hot_rank DESC, published DESC);
CREATE INDEX idx_post_aggregates_featured_community_scaled ON post_aggregates (community_id, featured_community DESC, scaled_rank DESC, published DESC);
CREATE INDEX idx_post_aggregates_featured_community_most_comments ON post_aggregates (community_id, featured_community DESC, comments DESC, published DESC);
CREATE INDEX idx_post_aggregates_featured_community_newest_comment_time ON post_aggregates (community_id, featured_community DESC, newest_comment_time DESC);
CREATE INDEX idx_post_aggregates_featured_community_newest_comment_time_necro ON post_aggregates (community_id, featured_community DESC, newest_comment_time_necro DESC);
CREATE INDEX idx_post_aggregates_featured_community_published ON post_aggregates (community_id, featured_community DESC, published DESC);
CREATE INDEX idx_post_aggregates_featured_community_score ON post_aggregates (community_id, featured_community DESC, score DESC, published DESC);