diff --git a/crates/db_schema/src/schema_setup.rs b/crates/db_schema/src/schema_setup.rs index 7caa31ab0..573c9195b 100644 --- a/crates/db_schema/src/schema_setup.rs +++ b/crates/db_schema/src/schema_setup.rs @@ -59,10 +59,14 @@ impl<'a, 'b> MigrationHarness for MigrationHarnessWrapper<'a, 'b> { #[cfg(test)] if self.options.enable_diff_check { - let before = diff_check::get_dump(); + let before = diff_check::get_dump(&mut self.conn); self.conn.run_migration(migration)?; self.conn.revert_migration(migration)?; - diff_check::check_dump_diff(before, &format!("migrations/{name}/down.sql")); + diff_check::check_dump_diff( + &mut self.conn, + before, + &format!("migrations/{name}/down.sql"), + ); } let start_time = Instant::now(); @@ -187,6 +191,7 @@ pub fn run(options: Options) -> LemmyResult<()> { } // Running without transaction allows pg_dump to see results of migrations + // TODO never use 1 transaction let run_in_transaction = !options.enable_diff_check; let transaction = |conn: &mut PgConnection| -> LemmyResult<()> { @@ -242,7 +247,7 @@ pub fn run(options: Options) -> LemmyResult<()> { if !(options.revert && !options.redo_after_revert) { #[cfg(test)] if options.enable_diff_check { - let before = diff_check::get_dump(); + let before = diff_check::get_dump(&mut wrapper.conn); // todo move replaceable_schema dir path to let/const? wrapper .conn @@ -252,7 +257,8 @@ pub fn run(options: Options) -> LemmyResult<()> { wrapper .conn .batch_execute("DROP SCHEMA IF EXISTS r CASCADE;")?; - diff_check::check_dump_diff(before, "replaceable_schema"); + // todo use different first output line in this case + diff_check::check_dump_diff(&mut wrapper.conn, before, "replaceable_schema"); } wrapper diff --git a/crates/db_schema/src/schema_setup/diff_check.rs b/crates/db_schema/src/schema_setup/diff_check.rs index 11c7142d8..af64815e5 100644 --- a/crates/db_schema/src/schema_setup/diff_check.rs +++ b/crates/db_schema/src/schema_setup/diff_check.rs @@ -1,3 +1,4 @@ +use diesel::{PgConnection, RunQueryDsl}; use lemmy_utils::settings::SETTINGS; use std::{ borrow::Cow, @@ -6,7 +7,21 @@ use std::{ process::{Command, Stdio}, }; -pub fn get_dump() -> String { +diesel::sql_function! { + fn pg_export_snapshot() -> diesel::sql_types::Text; +} + +pub fn get_dump(conn: &mut PgConnection) -> String { + /*// Required for pg_dump to see uncommitted changes from a different database connection + + // The pg_dump command runs independently from `conn`, which means it can't see changes from + // an uncommitted transaction. NASA made each migration run in a separate transaction. When + // it was discovered that + let snapshot = diesel::select(pg_export_snapshot()) + .get_result::(conn) + .expect("pg_export_snapshot failed"); + let snapshot_arg = format!("--snapshot={snapshot}");*/ + let output = Command::new("pg_dump") .args(["--schema-only"]) .env("DATABASE_URL", SETTINGS.get_database_url()) @@ -23,8 +38,8 @@ pub fn get_dump() -> String { const PATTERN_LEN: usize = 19; // TODO add unit test for output -pub fn check_dump_diff(mut before: String, name: &str) { - let mut after = get_dump(); +pub fn check_dump_diff(conn: &mut PgConnection, mut before: String, name: &str) { + let mut after = get_dump(conn); // Ignore timestamp differences by removing timestamps for dump in [&mut before, &mut after] { for index in 0.. { @@ -97,10 +112,12 @@ pub fn check_dump_diff(mut before: String, name: &str) { let (most_similar_chunk_index, (most_similar_chunk, _)) = only_in_after .iter() .enumerate() - .max_by_key(|(_, (_, after_chunk_filtered))| { + .max_by_key(|(_, (after_chunk, after_chunk_filtered))| { diff::chars(after_chunk_filtered, &before_chunk_filtered) .into_iter() - .filter(|i| matches!(i, diff::Result::Both(_, _))) + .filter(|i| matches!(i, diff::Result::Both(c, _) + // This increases accuracy for some trigger function diffs + if c.is_lowercase())) .count() }) .expect("resize should have prevented this from failing"); @@ -126,16 +143,18 @@ fn chunks<'a>(dump: &'a str) -> impl Iterator> { let mut remaining = dump; std::iter::from_fn(move || { remaining = remaining.trim_start(); - while remaining.starts_with("--") { - remaining = remaining.split_once('\n')?.1; - remaining = remaining.trim_start(); + while let Some(s) = remove_skipped_item_from_beginning(remaining) { + remaining = s.trim_start(); } // `a` can't be empty because of trim_start let (result, after_result) = remaining.split_once("\n\n")?; remaining = after_result; Some(if result.starts_with("CREATE TABLE ") { // Allow column order to change - let mut lines = result.lines().map(|line| line.strip_suffix(',').unwrap_or(line)).collect::>(); + let mut lines = result + .lines() + .map(|line| line.strip_suffix(',').unwrap_or(line)) + .collect::>(); lines.sort_unstable_by_key(|line| -> (u8, &str) { let placement = match line.chars().next() { Some('C') => 0, @@ -151,3 +170,18 @@ fn chunks<'a>(dump: &'a str) -> impl Iterator> { }) }) } + +fn remove_skipped_item_from_beginning(s: &str) -> Option<&str> { + // Skip commented line + if let Some(after) = s.strip_prefix("--") { + Some(after.split_once('\n').unwrap_or_default().1) + } + // Skip view definition that's replaced later (the first definition selects all nulls) + else if let Some(after) = s.strip_prefix("CREATE VIEW ") { + let (name, after_name) = after.split_once(' ').unwrap_or_default(); + Some(after_name.split_once("\n\n").unwrap_or_default().1) + .filter(|after_view| after_view.contains(&format!("\nCREATE OR REPLACE VIEW {name} "))) + } else { + None + } +} diff --git a/migrations/2020-04-07-135912_add_user_community_apub_constraints/down.sql b/migrations/2020-04-07-135912_add_user_community_apub_constraints/down.sql index 1b8b42ac0..2ed050ab0 100644 --- a/migrations/2020-04-07-135912_add_user_community_apub_constraints/down.sql +++ b/migrations/2020-04-07-135912_add_user_community_apub_constraints/down.sql @@ -2,9 +2,11 @@ DROP VIEW user_view CASCADE; ALTER TABLE user_ - ADD COLUMN fedi_name varchar(40) NOT NULL; + ADD COLUMN fedi_name varchar(40) NOT NULL DEFAULT 'http://fake.com'; ALTER TABLE user_ +-- Default is only for existing rows + ALTER COLUMN fedi_name DROP DEFAULT, ADD CONSTRAINT user__name_fedi_name_key UNIQUE (name, fedi_name); -- Community diff --git a/migrations/2020-04-14-163701_update_views_for_activitypub/down.sql b/migrations/2020-04-14-163701_update_views_for_activitypub/down.sql index 2e88d51fe..2434fe6c0 100644 --- a/migrations/2020-04-14-163701_update_views_for_activitypub/down.sql +++ b/migrations/2020-04-14-163701_update_views_for_activitypub/down.sql @@ -61,7 +61,17 @@ DROP VIEW community_aggregates_view CASCADE; CREATE VIEW community_aggregates_view AS SELECT - c.*, + c.id, + c.name, + c.title, + c.description, + c.category_id, + c.creator_id, + c.removed, + c.published, + c.updated, + c.deleted, + c.nsfw, ( SELECT name @@ -277,8 +287,23 @@ DROP VIEW post_aggregates_view; -- regen post view CREATE VIEW post_aggregates_view AS -SELECT - p.*, +SELECT p.id, + p.name, + p.url, + p.body, + p.creator_id, + p.community_id, + p.removed, + p.locked, + p.published, + p.updated, + p.deleted, + p.nsfw, + p.stickied, + p.embed_title, + p.embed_description, + p.embed_html, + p.thumbnail_url, ( SELECT u.banned @@ -511,7 +536,16 @@ DROP VIEW comment_aggregates_view; -- reply and comment view CREATE VIEW comment_aggregates_view AS SELECT - c.*, + c.id, + c.creator_id, + c.post_id, + c.parent_id, + c.content, + c.removed, + c.read, + c.published, + c.updated, + c.deleted, ( SELECT community_id diff --git a/migrations/2020-06-30-135809_remove_mat_views/down.sql b/migrations/2020-06-30-135809_remove_mat_views/down.sql index ea82e57b4..e0b2d0e14 100644 --- a/migrations/2020-06-30-135809_remove_mat_views/down.sql +++ b/migrations/2020-06-30-135809_remove_mat_views/down.sql @@ -966,3 +966,252 @@ FROM all_comment ac LEFT JOIN user_mention um ON um.comment_id = ac.id; +-- comment +CREATE OR REPLACE FUNCTION refresh_comment () + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ +BEGIN + REFRESH MATERIALIZED VIEW CONCURRENTLY post_aggregates_mview; + REFRESH MATERIALIZED VIEW CONCURRENTLY comment_aggregates_mview; + REFRESH MATERIALIZED VIEW CONCURRENTLY community_aggregates_mview; + REFRESH MATERIALIZED VIEW CONCURRENTLY user_mview; + RETURN NULL; +END +$$; + +CREATE OR REPLACE TRIGGER refresh_comment + AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON comment + FOR EACH statement + EXECUTE PROCEDURE refresh_comment (); + +-- comment_like +CREATE OR REPLACE FUNCTION refresh_comment_like () + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ +BEGIN + REFRESH MATERIALIZED VIEW CONCURRENTLY comment_aggregates_mview; + REFRESH MATERIALIZED VIEW CONCURRENTLY user_mview; + RETURN NULL; +END +$$; + +CREATE OR REPLACE TRIGGER refresh_comment_like + AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON comment_like + FOR EACH statement + EXECUTE PROCEDURE refresh_comment_like (); + +-- community_follower +CREATE OR REPLACE FUNCTION refresh_community_follower () + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ +BEGIN + REFRESH MATERIALIZED VIEW CONCURRENTLY community_aggregates_mview; + REFRESH MATERIALIZED VIEW CONCURRENTLY post_aggregates_mview; + RETURN NULL; +END +$$; + +CREATE or replace TRIGGER refresh_community_follower + AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON community_follower + FOR EACH statement + EXECUTE PROCEDURE refresh_community_follower (); + +-- community_user_ban +CREATE OR REPLACE FUNCTION refresh_community_user_ban () + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ +BEGIN + REFRESH MATERIALIZED VIEW CONCURRENTLY comment_aggregates_mview; + REFRESH MATERIALIZED VIEW CONCURRENTLY post_aggregates_mview; + RETURN NULL; +END +$$; + +CREATE or replace TRIGGER refresh_community_user_ban + AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON community_user_ban + FOR EACH statement + EXECUTE PROCEDURE refresh_community_user_ban (); + +-- post_like +CREATE OR REPLACE FUNCTION refresh_post_like () + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ +BEGIN + REFRESH MATERIALIZED VIEW CONCURRENTLY post_aggregates_mview; + REFRESH MATERIALIZED VIEW CONCURRENTLY user_mview; + RETURN NULL; +END +$$; + +CREATE or replace TRIGGER refresh_post_like + AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON post_like + FOR EACH statement + EXECUTE PROCEDURE refresh_post_like (); + +CREATE or replace VIEW community_moderator_view AS +SELECT + *, + ( + SELECT + actor_id + FROM + user_ u + WHERE + cm.user_id = u.id) AS user_actor_id, + ( + SELECT + local + FROM + user_ u + WHERE + cm.user_id = u.id) AS user_local, + ( + SELECT + name + FROM + user_ u + WHERE + cm.user_id = u.id) AS user_name, + ( + SELECT + avatar + FROM + user_ u + WHERE + cm.user_id = u.id), ( + SELECT + actor_id + FROM + community c + WHERE + cm.community_id = c.id) AS community_actor_id, + ( + SELECT + local + FROM + community c + WHERE + cm.community_id = c.id) AS community_local, + ( + SELECT + name + FROM + community c + WHERE + cm.community_id = c.id) AS community_name +FROM + community_moderator cm; + +CREATE or replace VIEW community_follower_view AS +SELECT + *, + ( + SELECT + actor_id + FROM + user_ u + WHERE + cf.user_id = u.id) AS user_actor_id, + ( + SELECT + local + FROM + user_ u + WHERE + cf.user_id = u.id) AS user_local, + ( + SELECT + name + FROM + user_ u + WHERE + cf.user_id = u.id) AS user_name, + ( + SELECT + avatar + FROM + user_ u + WHERE + cf.user_id = u.id), ( + SELECT + actor_id + FROM + community c + WHERE + cf.community_id = c.id) AS community_actor_id, + ( + SELECT + local + FROM + community c + WHERE + cf.community_id = c.id) AS community_local, + ( + SELECT + name + FROM + community c + WHERE + cf.community_id = c.id) AS community_name +FROM + community_follower cf; + +CREATE or replace VIEW community_user_ban_view AS +SELECT + *, + ( + SELECT + actor_id + FROM + user_ u + WHERE + cm.user_id = u.id) AS user_actor_id, + ( + SELECT + local + FROM + user_ u + WHERE + cm.user_id = u.id) AS user_local, + ( + SELECT + name + FROM + user_ u + WHERE + cm.user_id = u.id) AS user_name, + ( + SELECT + avatar + FROM + user_ u + WHERE + cm.user_id = u.id), ( + SELECT + actor_id + FROM + community c + WHERE + cm.community_id = c.id) AS community_actor_id, + ( + SELECT + local + FROM + community c + WHERE + cm.community_id = c.id) AS community_local, + ( + SELECT + name + FROM + community c + WHERE + cm.community_id = c.id) AS community_name +FROM + community_user_ban cm; + diff --git a/migrations/2020-07-08-202609_add_creator_published/down.sql b/migrations/2020-07-08-202609_add_creator_published/down.sql index fd1ca1cb4..034e7f7a0 100644 --- a/migrations/2020-07-08-202609_add_creator_published/down.sql +++ b/migrations/2020-07-08-202609_add_creator_published/down.sql @@ -493,3 +493,5 @@ SELECT FROM post_aggregates_fast pav; +CREATE INDEX idx_post_aggregates_fast_hot_rank_published ON post_aggregates_fast (hot_rank DESC, published DESC); + diff --git a/migrations/2020-10-07-234221_fix_fast_triggers/down.sql b/migrations/2020-10-07-234221_fix_fast_triggers/down.sql index 5a905564a..7697dbc6c 100644 --- a/migrations/2020-10-07-234221_fix_fast_triggers/down.sql +++ b/migrations/2020-10-07-234221_fix_fast_triggers/down.sql @@ -155,7 +155,8 @@ BEGIN UPDATE post_aggregates_fast AS paf SET - hot_rank = pav.hot_rank + hot_rank = pav.hot_rank, + hot_rank_active = pav.hot_rank_active FROM post_aggregates_view AS pav WHERE @@ -220,14 +221,36 @@ BEGIN post_aggregates_view WHERE id = NEW.post_id; - -- Force the hot rank as zero on week-older posts + -- Update the comment hot_ranks as of last week + UPDATE + comment_aggregates_fast AS caf + SET + hot_rank = cav.hot_rank, + hot_rank_active = cav.hot_rank_active + FROM + comment_aggregates_view AS cav + WHERE + caf.id = cav.id + AND (cav.published > ('now'::timestamp - '1 week'::interval)); + -- Update the post ranks UPDATE post_aggregates_fast AS paf SET - hot_rank = 0 + hot_rank = pav.hot_rank, + hot_rank_active = pav.hot_rank_active + FROM + post_aggregates_view AS pav + WHERE + paf.id = pav.id + AND (pav.published > ('now'::timestamp - '1 week'::interval)); + -- Force the hot rank active as zero on 2 day-older posts (necro-bump) + UPDATE + post_aggregates_fast AS paf + SET + hot_rank_active = 0 WHERE paf.id = NEW.post_id - AND (paf.published < ('now'::timestamp - '1 week'::interval)); + AND (paf.published < ('now'::timestamp - '2 days'::interval)); -- Update community number of comments UPDATE community_aggregates_fast AS caf