From ced9bb52168379834489f49716363a8054ed9c29 Mon Sep 17 00:00:00 2001 From: Dull Bananas Date: Mon, 20 May 2024 02:23:56 +0000 Subject: [PATCH] stuff --- .../db_schema/src/schema_setup/diff_check.rs | 49 ++++++++++++--- .../down.sql | 2 +- .../down.sql | 4 ++ .../down.sql | 61 +++++++++++++++++-- .../2021-09-20-112945_jwt-secret/down.sql | 2 + .../2022-07-07-182650_comment_ltrees/up.sql | 2 +- .../down.sql | 2 +- .../down.sql | 2 +- .../down.sql | 2 +- .../2023-07-24-232635_trigram-index/down.sql | 2 + .../2023-08-23-182533_scaled_rank/down.sql | 18 ++++++ .../2023-09-18-141700_login-token/down.sql | 2 +- .../down.sql | 2 +- 13 files changed, 130 insertions(+), 20 deletions(-) diff --git a/crates/db_schema/src/schema_setup/diff_check.rs b/crates/db_schema/src/schema_setup/diff_check.rs index f3b0f2aad..dd731c329 100644 --- a/crates/db_schema/src/schema_setup/diff_check.rs +++ b/crates/db_schema/src/schema_setup/diff_check.rs @@ -21,7 +21,6 @@ pub fn get_dump(conn: &mut PgConnection) -> String { .get_result::(conn) .expect("pg_export_snapshot failed"); let snapshot_arg = format!("--snapshot={snapshot}");*/ - let output = Command::new("pg_dump") .args(["--schema-only"]) .env("DATABASE_URL", SETTINGS.get_database_url()) @@ -40,6 +39,9 @@ const PATTERN_LEN: usize = 19; // TODO add unit test for output pub fn check_dump_diff(conn: &mut PgConnection, mut before: String, name: &str) { let mut after = get_dump(conn); + if after == before { + return; + } // Ignore timestamp differences by removing timestamps for dump in [&mut before, &mut after] { for index in 0.. { @@ -74,9 +76,6 @@ pub fn check_dump_diff(conn: &mut PgConnection, mut before: String, name: &str) } } - if after == before { - return; - } let [before_chunks, after_chunks] = [&before, &after].map(|dump| chunks(dump).collect::>()); @@ -102,8 +101,8 @@ pub fn check_dump_diff(conn: &mut PgConnection, mut before: String, name: &str) } let after_has_more = only_in_before.len() < only_in_after.len(); - // outer iterator in the loop below should be the one with empty strings, otherwise the empty strings - // would appear to have the most similarity + // outer iterator in the loop below should not be the one with empty strings, otherwise the empty strings + // would be equally similar to any other chunk let (chunks_gt, chunks_lt) = if after_has_more { only_in_before.resize_with(only_in_after.len(),Default::default); @@ -118,15 +117,15 @@ pub fn check_dump_diff(conn: &mut PgConnection, mut before: String, name: &str) for (before_chunk, before_chunk_filtered) in chunks_lt { let default = Default::default(); //panic!("{:?}",(before_chunk.clone(),chunks_lt.clone())); - let (most_similar_chunk_index, (most_similar_chunk, _)) = chunks_gt + let (most_similar_chunk_index, (most_similar_chunk, most_similar_chunk_filtered)) = chunks_gt .iter() .enumerate() .max_by_key(|(_, (after_chunk, after_chunk_filtered))| { diff::chars(after_chunk_filtered, &before_chunk_filtered) .into_iter() .filter(|i| matches!(i, diff::Result::Both(c, _) - // This increases accuracy for some trigger function diffs - if c.is_lowercase())) + // `is_lowercase` increases accuracy for some trigger function diffs + if c.is_lowercase() || c.is_numeric())) .count() }) .unwrap_or((0,&default)); @@ -143,6 +142,7 @@ pub fn check_dump_diff(conn: &mut PgConnection, mut before: String, name: &str) } .expect("failed to build string"); } + write!(&mut output, "\n{most_similar_chunk_filtered}"); if !chunks_gt.is_empty() { chunks_gt.swap_remove(most_similar_chunk_index);} } @@ -178,6 +178,37 @@ fn chunks<'a>(dump: &'a str) -> impl Iterator> { (placement, line) }); Cow::Owned(lines.join("\n")) + } else if result.starts_with("CREATE VIEW") || result.starts_with("CREATE OR REPLACE VIEW") { + // Allow column order to change + let is_simple_select_statement = result + .lines() + .enumerate() + .all(|(i, mut line)| { + line = line.trim_start(); + match (i, line.chars().next()) { + (0, Some('C')) => true, // create + (1, Some('S')) => true, // select + (_, Some('F')) if line.ends_with(';') => true, // from + (_, Some(c)) if c.is_lowercase() => true, // column name + _ => false + } + }); + if is_simple_select_statement { + let mut lines = result + .lines() + .map(|line| line.strip_suffix(',').unwrap_or(line)) + .collect::>(); + lines.sort_unstable_by_key(|line| -> (u8, &str) { + let placement = match line.trim_start().chars().next() { + Some('C') => 0, + Some('S') => 1, + Some('F') => 3, + _ => 2, + }; + (placement, line) + }); + Cow::Owned(lines.join("\n")) + }else{Cow::Borrowed(result)} } else { Cow::Borrowed(result) }) diff --git a/migrations/2020-11-05-152724_activity_remove_user_id/down.sql b/migrations/2020-11-05-152724_activity_remove_user_id/down.sql index 580c93e0d..2553862ed 100644 --- a/migrations/2020-11-05-152724_activity_remove_user_id/down.sql +++ b/migrations/2020-11-05-152724_activity_remove_user_id/down.sql @@ -1,5 +1,5 @@ ALTER TABLE activity - ADD COLUMN user_id INTEGER; + ADD COLUMN user_id INTEGER NOT NULL REFERENCES user_(id) ON UPDATE CASCADE ON DELETE CASCADE; ALTER TABLE activity DROP COLUMN sensitive; diff --git a/migrations/2021-02-25-112959_remove-categories/down.sql b/migrations/2021-02-25-112959_remove-categories/down.sql index fb9e69232..6a7fe7cd1 100644 --- a/migrations/2021-02-25-112959_remove-categories/down.sql +++ b/migrations/2021-02-25-112959_remove-categories/down.sql @@ -34,5 +34,9 @@ INSERT INTO category (name) ALTER TABLE community ADD category_id int REFERENCES category ON UPDATE CASCADE ON DELETE CASCADE NOT NULL DEFAULT 1; +-- Default is only for existing rows +ALTER TABLE community + ALTER COLUMN category_id DROP DEFAULT; + CREATE INDEX idx_community_category ON community (category_id); diff --git a/migrations/2021-03-09-171136_split_user_table_2/down.sql b/migrations/2021-03-09-171136_split_user_table_2/down.sql index d9fbf0718..bf1708b4a 100644 --- a/migrations/2021-03-09-171136_split_user_table_2/down.sql +++ b/migrations/2021-03-09-171136_split_user_table_2/down.sql @@ -229,7 +229,7 @@ ALTER SEQUENCE person_id_seq -- Add the columns back in ALTER TABLE user_ ADD COLUMN password_encrypted text NOT NULL DEFAULT 'changeme', - ADD COLUMN email text, + ADD COLUMN email text UNIQUE, ADD COLUMN admin boolean DEFAULT FALSE NOT NULL, ADD COLUMN show_nsfw boolean DEFAULT FALSE NOT NULL, ADD COLUMN theme character varying(20) DEFAULT 'darkly'::character varying NOT NULL, @@ -238,7 +238,10 @@ ALTER TABLE user_ ADD COLUMN lang character varying(20) DEFAULT 'browser'::character varying NOT NULL, ADD COLUMN show_avatars boolean DEFAULT TRUE NOT NULL, ADD COLUMN send_notifications_to_email boolean DEFAULT FALSE NOT NULL, - ADD COLUMN matrix_user_id text; + ADD COLUMN matrix_user_id text UNIQUE; + +-- Default is only for existing rows +alter table user_ alter column password_encrypted drop default; -- Update the user_ table with the local_user data UPDATE @@ -264,13 +267,63 @@ CREATE UNIQUE INDEX idx_user_email_lower ON user_ (lower(email)); CREATE VIEW user_alias_1 AS SELECT - * + id, + actor_id, + admin, + avatar, + banned, + banner, + bio, + default_listing_type, + default_sort_type, + deleted, + email, + lang, + last_refreshed_at, + local, + matrix_user_id, + name, + password_encrypted, + preferred_username, + private_key, + public_key, + published, + send_notifications_to_email, + show_avatars, + show_nsfw, + theme, + updated FROM user_; CREATE VIEW user_alias_2 AS SELECT - * + id, + actor_id, + admin, + avatar, + banned, + banner, + bio, + default_listing_type, + default_sort_type, + deleted, + email, + lang, + last_refreshed_at, + local, + matrix_user_id, + name, + password_encrypted, + preferred_username, + private_key, + public_key, + published, + send_notifications_to_email, + show_avatars, + show_nsfw, + theme, + updated FROM user_; diff --git a/migrations/2021-09-20-112945_jwt-secret/down.sql b/migrations/2021-09-20-112945_jwt-secret/down.sql index 89350d96a..a633110a9 100644 --- a/migrations/2021-09-20-112945_jwt-secret/down.sql +++ b/migrations/2021-09-20-112945_jwt-secret/down.sql @@ -1,2 +1,4 @@ DROP TABLE secret; +DROP EXTENSION pgcrypto; + diff --git a/migrations/2022-07-07-182650_comment_ltrees/up.sql b/migrations/2022-07-07-182650_comment_ltrees/up.sql index c0ef15ee7..73c00e439 100644 --- a/migrations/2022-07-07-182650_comment_ltrees/up.sql +++ b/migrations/2022-07-07-182650_comment_ltrees/up.sql @@ -182,7 +182,7 @@ CREATE INDEX idx_path_gist ON comment USING gist (path); -- Drop the parent_id column ALTER TABLE comment - DROP COLUMN parent_id CASCADE,add column if not exists poop int;alter table post add column if not exists poop int; + DROP COLUMN parent_id CASCADE; ALTER TABLE comment ENABLE TRIGGER USER; diff --git a/migrations/2022-10-06-183632_move_blocklist_to_db/down.sql b/migrations/2022-10-06-183632_move_blocklist_to_db/down.sql index af95b8773..91ad3fc8b 100644 --- a/migrations/2022-10-06-183632_move_blocklist_to_db/down.sql +++ b/migrations/2022-10-06-183632_move_blocklist_to_db/down.sql @@ -6,7 +6,7 @@ ALTER TABLE site ADD COLUMN community_creation_admin_only boolean DEFAULT FALSE NOT NULL, ADD COLUMN require_email_verification boolean DEFAULT FALSE NOT NULL, ADD COLUMN require_application boolean DEFAULT TRUE NOT NULL, - ADD COLUMN application_question text DEFAULT 'to verify that you are human, please explain why you want to create an account on this site'::text, + ADD COLUMN application_question text DEFAULT 'To verify that you are human, please explain why you want to create an account on this site'::text, ADD COLUMN private_instance boolean DEFAULT FALSE NOT NULL, ADD COLUMN default_theme text DEFAULT 'browser'::text NOT NULL, ADD COLUMN default_post_listing_type text DEFAULT 'Local'::text NOT NULL, diff --git a/migrations/2023-02-05-102549_drop-site-federation-debug/down.sql b/migrations/2023-02-05-102549_drop-site-federation-debug/down.sql index 4e141e423..90844769a 100644 --- a/migrations/2023-02-05-102549_drop-site-federation-debug/down.sql +++ b/migrations/2023-02-05-102549_drop-site-federation-debug/down.sql @@ -1,3 +1,3 @@ ALTER TABLE local_site - ADD COLUMN federation_debug int DEFAULT 0; + ADD COLUMN federation_debug boolean DEFAULT false not null; diff --git a/migrations/2023-04-14-175955_add_listingtype_sorttype_enums/down.sql b/migrations/2023-04-14-175955_add_listingtype_sorttype_enums/down.sql index ceb36f538..cdd2f573b 100644 --- a/migrations/2023-04-14-175955_add_listingtype_sorttype_enums/down.sql +++ b/migrations/2023-04-14-175955_add_listingtype_sorttype_enums/down.sql @@ -117,7 +117,7 @@ ALTER TABLE local_site ALTER COLUMN default_post_listing_type TYPE text; ALTER TABLE local_site - ALTER COLUMN default_post_listing_type SET DEFAULT 1; + ALTER COLUMN default_post_listing_type SET DEFAULT 'Local'; -- Drop the types DROP TYPE listing_type_enum; diff --git a/migrations/2023-07-24-232635_trigram-index/down.sql b/migrations/2023-07-24-232635_trigram-index/down.sql index 6883178f8..f3e3d95ad 100644 --- a/migrations/2023-07-24-232635_trigram-index/down.sql +++ b/migrations/2023-07-24-232635_trigram-index/down.sql @@ -6,3 +6,5 @@ DROP INDEX idx_person_trigram; DROP INDEX idx_community_trigram; +DROP EXTENSION pg_trgm; + diff --git a/migrations/2023-08-23-182533_scaled_rank/down.sql b/migrations/2023-08-23-182533_scaled_rank/down.sql index afe1bfc21..85a0e3eef 100644 --- a/migrations/2023-08-23-182533_scaled_rank/down.sql +++ b/migrations/2023-08-23-182533_scaled_rank/down.sql @@ -85,3 +85,21 @@ ALTER TABLE local_user -- drop the old enum DROP TYPE sort_type_enum__; +-- Remove int to float conversions that were automatically added to index filters +DROP INDEX +idx_comment_aggregates_nonzero_hotrank, +idx_community_aggregates_nonzero_hotrank, +idx_post_aggregates_nonzero_hotrank; + +CREATE INDEX idx_community_aggregates_nonzero_hotrank ON community_aggregates (published) +WHERE + hot_rank != 0; + +CREATE INDEX idx_comment_aggregates_nonzero_hotrank ON comment_aggregates (published) +WHERE + hot_rank != 0; + +CREATE INDEX idx_post_aggregates_nonzero_hotrank ON post_aggregates (published DESC) +WHERE + hot_rank != 0 OR hot_rank_active != 0; + diff --git a/migrations/2023-09-18-141700_login-token/down.sql b/migrations/2023-09-18-141700_login-token/down.sql index 826e05e70..48ecd1e1e 100644 --- a/migrations/2023-09-18-141700_login-token/down.sql +++ b/migrations/2023-09-18-141700_login-token/down.sql @@ -1,5 +1,5 @@ DROP TABLE login_token; ALTER TABLE local_user - ADD COLUMN validator_time timestamp NOT NULL DEFAULT now(); + ADD COLUMN validator_time timestamptz NOT NULL DEFAULT now(); diff --git a/migrations/2023-10-17-181800_drop_remove_community_expires/down.sql b/migrations/2023-10-17-181800_drop_remove_community_expires/down.sql index 048c3d7f1..d0e792db5 100644 --- a/migrations/2023-10-17-181800_drop_remove_community_expires/down.sql +++ b/migrations/2023-10-17-181800_drop_remove_community_expires/down.sql @@ -1,3 +1,3 @@ ALTER TABLE mod_remove_community - ADD COLUMN expires timestamp; + ADD COLUMN expires timestamptz;