stuff

crates/db_schema/src/schema_setup.rs

@@ -59,10 +59,14 @@ impl<'a, 'b> MigrationHarness<Pg> for MigrationHarnessWrapper<'a, 'b> {
 
     #[cfg(test)]
     if self.options.enable_diff_check {
-      let before = diff_check::get_dump();
+      let before = diff_check::get_dump(&mut self.conn);
       self.conn.run_migration(migration)?;
       self.conn.revert_migration(migration)?;
-      diff_check::check_dump_diff(before, &format!("migrations/{name}/down.sql"));
+      diff_check::check_dump_diff(
+        &mut self.conn,
+        before,
+        &format!("migrations/{name}/down.sql"),
+      );
     }
 
     let start_time = Instant::now();
@@ -187,6 +191,7 @@ pub fn run(options: Options) -> LemmyResult<()> {
   }
 
   // Running without transaction allows pg_dump to see results of migrations
+  // TODO never use 1 transaction
   let run_in_transaction = !options.enable_diff_check;
 
   let transaction = |conn: &mut PgConnection| -> LemmyResult<()> {
@@ -242,7 +247,7 @@ pub fn run(options: Options) -> LemmyResult<()> {
     if !(options.revert && !options.redo_after_revert) {
       #[cfg(test)]
       if options.enable_diff_check {
-        let before = diff_check::get_dump();
+        let before = diff_check::get_dump(&mut wrapper.conn);
         // todo move replaceable_schema dir path to let/const?
         wrapper
           .conn
@@ -252,7 +257,8 @@ pub fn run(options: Options) -> LemmyResult<()> {
         wrapper
           .conn
           .batch_execute("DROP SCHEMA IF EXISTS r CASCADE;")?;
-        diff_check::check_dump_diff(before, "replaceable_schema");
+        // todo use different first output line in this case
+        diff_check::check_dump_diff(&mut wrapper.conn, before, "replaceable_schema");
       }
 
       wrapper

crates/db_schema/src/schema_setup/diff_check.rs

@@ -1,3 +1,4 @@
+use diesel::{PgConnection, RunQueryDsl};
 use lemmy_utils::settings::SETTINGS;
 use std::{
   borrow::Cow,
@@ -6,7 +7,21 @@ use std::{
   process::{Command, Stdio},
 };
 
-pub fn get_dump() -> String {
+diesel::sql_function! {
+  fn pg_export_snapshot() -> diesel::sql_types::Text;
+}
+
+pub fn get_dump(conn: &mut PgConnection) -> String {
+  /*// Required for pg_dump to see uncommitted changes from a different database connection
+
+  // The pg_dump command runs independently from `conn`, which means it can't see changes from
+  // an uncommitted transaction. NASA made each migration run in a separate transaction. When
+  // it was discovered that 
+  let snapshot = diesel::select(pg_export_snapshot())
+    .get_result::<String>(conn)
+    .expect("pg_export_snapshot failed");
+  let snapshot_arg = format!("--snapshot={snapshot}");*/
+
   let output = Command::new("pg_dump")
     .args(["--schema-only"])
     .env("DATABASE_URL", SETTINGS.get_database_url())
@@ -23,8 +38,8 @@ pub fn get_dump() -> String {
 const PATTERN_LEN: usize = 19;
 
 // TODO add unit test for output
-pub fn check_dump_diff(mut before: String, name: &str) {
+pub fn check_dump_diff(conn: &mut PgConnection, mut before: String, name: &str) {
-  let mut after = get_dump();
+  let mut after = get_dump(conn);
   // Ignore timestamp differences by removing timestamps
   for dump in [&mut before, &mut after] {
     for index in 0.. {
@@ -97,10 +112,12 @@ pub fn check_dump_diff(mut before: String, name: &str) {
     let (most_similar_chunk_index, (most_similar_chunk, _)) = only_in_after
       .iter()
       .enumerate()
-      .max_by_key(|(_, (_, after_chunk_filtered))| {
+      .max_by_key(|(_, (after_chunk, after_chunk_filtered))| {
         diff::chars(after_chunk_filtered, &before_chunk_filtered)
           .into_iter()
-          .filter(|i| matches!(i, diff::Result::Both(_, _)))
+          .filter(|i| matches!(i, diff::Result::Both(c, _)
+          // This increases accuracy for some trigger function diffs
+          if c.is_lowercase()))
           .count()
       })
       .expect("resize should have prevented this from failing");
@@ -126,16 +143,18 @@ fn chunks<'a>(dump: &'a str) -> impl Iterator<Item = Cow<'a, str>> {
   let mut remaining = dump;
   std::iter::from_fn(move || {
     remaining = remaining.trim_start();
-    while remaining.starts_with("--") {
+    while let Some(s) = remove_skipped_item_from_beginning(remaining) {
-      remaining = remaining.split_once('\n')?.1;
+      remaining = s.trim_start();
-      remaining = remaining.trim_start();
     }
     // `a` can't be empty because of trim_start
     let (result, after_result) = remaining.split_once("\n\n")?;
     remaining = after_result;
     Some(if result.starts_with("CREATE TABLE ") {
       // Allow column order to change
-      let mut lines = result.lines().map(|line| line.strip_suffix(',').unwrap_or(line)).collect::<Vec<_>>();
+      let mut lines = result
+        .lines()
+        .map(|line| line.strip_suffix(',').unwrap_or(line))
+        .collect::<Vec<_>>();
       lines.sort_unstable_by_key(|line| -> (u8, &str) {
         let placement = match line.chars().next() {
           Some('C') => 0,
@@ -151,3 +170,18 @@ fn chunks<'a>(dump: &'a str) -> impl Iterator<Item = Cow<'a, str>> {
     })
   })
 }
+
+fn remove_skipped_item_from_beginning(s: &str) -> Option<&str> {
+  // Skip commented line
+  if let Some(after) = s.strip_prefix("--") {
+    Some(after.split_once('\n').unwrap_or_default().1)
+  }
+  // Skip view definition that's replaced later (the first definition selects all nulls)
+  else if let Some(after) = s.strip_prefix("CREATE VIEW ") {
+    let (name, after_name) = after.split_once(' ').unwrap_or_default();
+    Some(after_name.split_once("\n\n").unwrap_or_default().1)
+      .filter(|after_view| after_view.contains(&format!("\nCREATE OR REPLACE VIEW {name} ")))
+  } else {
+    None
+  }
+}

migrations/2020-04-07-135912_add_user_community_apub_constraints/down.sql

@@ -2,9 +2,11 @@
 DROP VIEW user_view CASCADE;
 
 ALTER TABLE user_
-    ADD COLUMN fedi_name varchar(40) NOT NULL;
+    ADD COLUMN fedi_name varchar(40) NOT NULL DEFAULT 'http://fake.com';
 
 ALTER TABLE user_
+-- Default is only for existing rows
+    ALTER COLUMN fedi_name DROP DEFAULT,
     ADD CONSTRAINT user__name_fedi_name_key UNIQUE (name, fedi_name);
 
 -- Community

migrations/2020-04-14-163701_update_views_for_activitypub/down.sql

@@ -61,7 +61,17 @@ DROP VIEW community_aggregates_view CASCADE;
 
 CREATE VIEW community_aggregates_view AS
 SELECT
-    c.*,
+    c.id,
+      c.name,
+      c.title,
+      c.description,
+      c.category_id,
+      c.creator_id,
+      c.removed,
+      c.published,
+      c.updated,
+      c.deleted,
+      c.nsfw,
     (
         SELECT
             name
@@ -277,8 +287,23 @@ DROP VIEW post_aggregates_view;
 
 -- regen post view
 CREATE VIEW post_aggregates_view AS
-SELECT
+SELECT p.id,
-    p.*,
+    p.name,
+      p.url,
+      p.body,
+      p.creator_id,
+      p.community_id,
+      p.removed,
+      p.locked,
+      p.published,
+      p.updated,
+      p.deleted,
+      p.nsfw,
+      p.stickied,
+      p.embed_title,
+      p.embed_description,
+      p.embed_html,
+      p.thumbnail_url,
     (
         SELECT
             u.banned
@@ -511,7 +536,16 @@ DROP VIEW comment_aggregates_view;
 -- reply and comment view
 CREATE VIEW comment_aggregates_view AS
 SELECT
-    c.*,
+    c.id,
+      c.creator_id,
+      c.post_id,
+      c.parent_id,
+      c.content,
+      c.removed,
+      c.read,
+      c.published,
+      c.updated,
+      c.deleted,
     (
         SELECT
             community_id

migrations/2020-06-30-135809_remove_mat_views/down.sql

@@ -966,3 +966,252 @@ FROM
     all_comment ac
     LEFT JOIN user_mention um ON um.comment_id = ac.id;
 
+-- comment
+CREATE OR REPLACE FUNCTION refresh_comment ()
+    RETURNS TRIGGER
+    LANGUAGE plpgsql
+    AS $$
+BEGIN
+    REFRESH MATERIALIZED VIEW CONCURRENTLY post_aggregates_mview;
+    REFRESH MATERIALIZED VIEW CONCURRENTLY comment_aggregates_mview;
+    REFRESH MATERIALIZED VIEW CONCURRENTLY community_aggregates_mview;
+    REFRESH MATERIALIZED VIEW CONCURRENTLY user_mview;
+    RETURN NULL;
+END
+$$;
+
+CREATE OR REPLACE TRIGGER refresh_comment
+    AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON comment
+    FOR EACH statement
+    EXECUTE PROCEDURE refresh_comment ();
+
+-- comment_like
+CREATE OR REPLACE FUNCTION refresh_comment_like ()
+    RETURNS TRIGGER
+    LANGUAGE plpgsql
+    AS $$
+BEGIN
+    REFRESH MATERIALIZED VIEW CONCURRENTLY comment_aggregates_mview;
+    REFRESH MATERIALIZED VIEW CONCURRENTLY user_mview;
+    RETURN NULL;
+END
+$$;
+
+CREATE OR REPLACE TRIGGER refresh_comment_like
+    AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON comment_like
+    FOR EACH statement
+    EXECUTE PROCEDURE refresh_comment_like ();
+
+-- community_follower
+CREATE OR REPLACE FUNCTION refresh_community_follower ()
+    RETURNS TRIGGER
+    LANGUAGE plpgsql
+    AS $$
+BEGIN
+    REFRESH MATERIALIZED VIEW CONCURRENTLY community_aggregates_mview;
+    REFRESH MATERIALIZED VIEW CONCURRENTLY post_aggregates_mview;
+    RETURN NULL;
+END
+$$;
+
+CREATE or replace TRIGGER refresh_community_follower
+    AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON community_follower
+    FOR EACH statement
+    EXECUTE PROCEDURE refresh_community_follower ();
+
+-- community_user_ban
+CREATE OR REPLACE FUNCTION refresh_community_user_ban ()
+    RETURNS TRIGGER
+    LANGUAGE plpgsql
+    AS $$
+BEGIN
+    REFRESH MATERIALIZED VIEW CONCURRENTLY comment_aggregates_mview;
+    REFRESH MATERIALIZED VIEW CONCURRENTLY post_aggregates_mview;
+    RETURN NULL;
+END
+$$;
+
+CREATE or replace TRIGGER refresh_community_user_ban
+    AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON community_user_ban
+    FOR EACH statement
+    EXECUTE PROCEDURE refresh_community_user_ban ();
+
+-- post_like
+CREATE OR REPLACE FUNCTION refresh_post_like ()
+    RETURNS TRIGGER
+    LANGUAGE plpgsql
+    AS $$
+BEGIN
+    REFRESH MATERIALIZED VIEW CONCURRENTLY post_aggregates_mview;
+    REFRESH MATERIALIZED VIEW CONCURRENTLY user_mview;
+    RETURN NULL;
+END
+$$;
+
+CREATE or replace TRIGGER refresh_post_like
+    AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON post_like
+    FOR EACH statement
+    EXECUTE PROCEDURE refresh_post_like ();
+
+CREATE or replace VIEW community_moderator_view AS
+SELECT
+    *,
+    (
+        SELECT
+            actor_id
+        FROM
+            user_ u
+        WHERE
+            cm.user_id = u.id) AS user_actor_id,
+    (
+        SELECT
+            local
+        FROM
+            user_ u
+        WHERE
+            cm.user_id = u.id) AS user_local,
+    (
+        SELECT
+            name
+        FROM
+            user_ u
+        WHERE
+            cm.user_id = u.id) AS user_name,
+    (
+        SELECT
+            avatar
+        FROM
+            user_ u
+        WHERE
+            cm.user_id = u.id), (
+        SELECT
+            actor_id
+        FROM
+            community c
+        WHERE
+            cm.community_id = c.id) AS community_actor_id,
+    (
+        SELECT
+            local
+        FROM
+            community c
+        WHERE
+            cm.community_id = c.id) AS community_local,
+    (
+        SELECT
+            name
+        FROM
+            community c
+        WHERE
+            cm.community_id = c.id) AS community_name
+FROM
+    community_moderator cm;
+
+CREATE or replace VIEW community_follower_view AS
+SELECT
+    *,
+    (
+        SELECT
+            actor_id
+        FROM
+            user_ u
+        WHERE
+            cf.user_id = u.id) AS user_actor_id,
+    (
+        SELECT
+            local
+        FROM
+            user_ u
+        WHERE
+            cf.user_id = u.id) AS user_local,
+    (
+        SELECT
+            name
+        FROM
+            user_ u
+        WHERE
+            cf.user_id = u.id) AS user_name,
+    (
+        SELECT
+            avatar
+        FROM
+            user_ u
+        WHERE
+            cf.user_id = u.id), (
+        SELECT
+            actor_id
+        FROM
+            community c
+        WHERE
+            cf.community_id = c.id) AS community_actor_id,
+    (
+        SELECT
+            local
+        FROM
+            community c
+        WHERE
+            cf.community_id = c.id) AS community_local,
+    (
+        SELECT
+            name
+        FROM
+            community c
+        WHERE
+            cf.community_id = c.id) AS community_name
+FROM
+    community_follower cf;
+
+CREATE or replace VIEW community_user_ban_view AS
+SELECT
+    *,
+    (
+        SELECT
+            actor_id
+        FROM
+            user_ u
+        WHERE
+            cm.user_id = u.id) AS user_actor_id,
+    (
+        SELECT
+            local
+        FROM
+            user_ u
+        WHERE
+            cm.user_id = u.id) AS user_local,
+    (
+        SELECT
+            name
+        FROM
+            user_ u
+        WHERE
+            cm.user_id = u.id) AS user_name,
+    (
+        SELECT
+            avatar
+        FROM
+            user_ u
+        WHERE
+            cm.user_id = u.id), (
+        SELECT
+            actor_id
+        FROM
+            community c
+        WHERE
+            cm.community_id = c.id) AS community_actor_id,
+    (
+        SELECT
+            local
+        FROM
+            community c
+        WHERE
+            cm.community_id = c.id) AS community_local,
+    (
+        SELECT
+            name
+        FROM
+            community c
+        WHERE
+            cm.community_id = c.id) AS community_name
+FROM
+    community_user_ban cm;
+

migrations/2020-07-08-202609_add_creator_published/down.sql

@@ -493,3 +493,5 @@ SELECT
 FROM
     post_aggregates_fast pav;
 
+CREATE INDEX idx_post_aggregates_fast_hot_rank_published ON post_aggregates_fast (hot_rank DESC, published DESC);
+

migrations/2020-10-07-234221_fix_fast_triggers/down.sql

@@ -155,7 +155,8 @@ BEGIN
         UPDATE
             post_aggregates_fast AS paf
         SET
-            hot_rank = pav.hot_rank
+            hot_rank = pav.hot_rank,
+            hot_rank_active = pav.hot_rank_active
         FROM
             post_aggregates_view AS pav
         WHERE
@@ -220,14 +221,36 @@ BEGIN
             post_aggregates_view
         WHERE
             id = NEW.post_id;
-        -- Force the hot rank as zero on week-older posts
+        -- Update the comment hot_ranks as of last week
+        UPDATE
+            comment_aggregates_fast AS caf
+        SET
+            hot_rank = cav.hot_rank,
+            hot_rank_active = cav.hot_rank_active
+        FROM
+            comment_aggregates_view AS cav
+        WHERE
+            caf.id = cav.id
+            AND (cav.published > ('now'::timestamp - '1 week'::interval));
+        -- Update the post ranks
         UPDATE
             post_aggregates_fast AS paf
         SET
-            hot_rank = 0
+            hot_rank = pav.hot_rank,
+            hot_rank_active = pav.hot_rank_active
+        FROM
+            post_aggregates_view AS pav
+        WHERE
+            paf.id = pav.id
+            AND (pav.published > ('now'::timestamp - '1 week'::interval));
+        -- Force the hot rank active as zero on 2 day-older posts (necro-bump)
+        UPDATE
+            post_aggregates_fast AS paf
+        SET
+            hot_rank_active = 0
         WHERE
             paf.id = NEW.post_id
-            AND (paf.published < ('now'::timestamp - '1 week'::interval));
+            AND (paf.published < ('now'::timestamp - '2 days'::interval));
         -- Update community number of comments
         UPDATE
             community_aggregates_fast AS caf