LemmyNet/lemmy
2
0
Fork 0
mirror of https://github.com/LemmyNet/lemmy.git synced 2024-12-28 13:51:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

stuff

Browse source
This commit is contained in:
Dull Bananas 2024-05-19 04:49:30 +00:00
parent 80ca94e176
commit 06b01ffecf
7 changed files with 372 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
crates/db_schema/src/schema_setup.rs
View file

@ -59,10 +59,14 @@ impl<'a, 'b> MigrationHarness<Pg> for MigrationHarnessWrapper<'a, 'b> {
#[cfg(test)]
if self.options.enable_diff_check {
let before = diff_check::get_dump();
let before = diff_check::get_dump(&mut self.conn);
self.conn.run_migration(migration)?;
self.conn.revert_migration(migration)?;
diff_check::check_dump_diff(before, &format!("migrations/{name}/down.sql"));
diff_check::check_dump_diff(
&mut self.conn,
before,
&format!("migrations/{name}/down.sql"),
);
}
let start_time = Instant::now();
@ -187,6 +191,7 @@ pub fn run(options: Options) -> LemmyResult<()> {
}
// Running without transaction allows pg_dump to see results of migrations
// TODO never use 1 transaction
let run_in_transaction = !options.enable_diff_check;
let transaction = |conn: &mut PgConnection| -> LemmyResult<()> {
@ -242,7 +247,7 @@ pub fn run(options: Options) -> LemmyResult<()> {
if !(options.revert && !options.redo_after_revert) {
#[cfg(test)]
if options.enable_diff_check {
let before = diff_check::get_dump();
let before = diff_check::get_dump(&mut wrapper.conn);
// todo move replaceable_schema dir path to let/const?
wrapper
.conn
@ -252,7 +257,8 @@ pub fn run(options: Options) -> LemmyResult<()> {
wrapper
.conn
.batch_execute("DROP SCHEMA IF EXISTS r CASCADE;")?;
diff_check::check_dump_diff(before, "replaceable_schema");
// todo use different first output line in this case
diff_check::check_dump_diff(&mut wrapper.conn, before, "replaceable_schema");
}
wrapper

52
crates/db_schema/src/schema_setup/diff_check.rs
View file

@ -1,3 +1,4 @@
use diesel::{PgConnection, RunQueryDsl};
use lemmy_utils::settings::SETTINGS;
use std::{
borrow::Cow,
@ -6,7 +7,21 @@ use std::{
process::{Command, Stdio},
};
pub fn get_dump() -> String {
diesel::sql_function! {
fn pg_export_snapshot() -> diesel::sql_types::Text;
}
pub fn get_dump(conn: &mut PgConnection) -> String {
/*// Required for pg_dump to see uncommitted changes from a different database connection
// The pg_dump command runs independently from `conn`, which means it can't see changes from
// an uncommitted transaction. NASA made each migration run in a separate transaction. When
// it was discovered that
let snapshot = diesel::select(pg_export_snapshot())
.get_result::<String>(conn)
.expect("pg_export_snapshot failed");
let snapshot_arg = format!("--snapshot={snapshot}");*/
let output = Command::new("pg_dump")
.args(["--schema-only"])
.env("DATABASE_URL", SETTINGS.get_database_url())
@ -23,8 +38,8 @@ pub fn get_dump() -> String {
const PATTERN_LEN: usize = 19;
// TODO add unit test for output
pub fn check_dump_diff(mut before: String, name: &str) {
let mut after = get_dump();
pub fn check_dump_diff(conn: &mut PgConnection, mut before: String, name: &str) {
let mut after = get_dump(conn);
// Ignore timestamp differences by removing timestamps
for dump in [&mut before, &mut after] {
for index in 0.. {
@ -97,10 +112,12 @@ pub fn check_dump_diff(mut before: String, name: &str) {
let (most_similar_chunk_index, (most_similar_chunk, _)) = only_in_after
.iter()
.enumerate()
.max_by_key(|(_, (_, after_chunk_filtered))| {
.max_by_key(|(_, (after_chunk, after_chunk_filtered))| {
diff::chars(after_chunk_filtered, &before_chunk_filtered)
.into_iter()
.filter(|i| matches!(i, diff::Result::Both(_, _)))
.filter(|i| matches!(i, diff::Result::Both(c, _)
// This increases accuracy for some trigger function diffs
if c.is_lowercase()))
.count()
})
.expect("resize should have prevented this from failing");
@ -126,16 +143,18 @@ fn chunks<'a>(dump: &'a str) -> impl Iterator<Item = Cow<'a, str>> {
let mut remaining = dump;
std::iter::from_fn(move || {
remaining = remaining.trim_start();
while remaining.starts_with("--") {
remaining = remaining.split_once('\n')?.1;
remaining = remaining.trim_start();
while let Some(s) = remove_skipped_item_from_beginning(remaining) {
remaining = s.trim_start();
}
// `a` can't be empty because of trim_start
let (result, after_result) = remaining.split_once("\n\n")?;
remaining = after_result;
Some(if result.starts_with("CREATE TABLE ") {
// Allow column order to change
let mut lines = result.lines().map(|line| line.strip_suffix(',').unwrap_or(line)).collect::<Vec<_>>();
let mut lines = result
.lines()
.map(|line| line.strip_suffix(',').unwrap_or(line))
.collect::<Vec<_>>();
lines.sort_unstable_by_key(|line| -> (u8, &str) {
let placement = match line.chars().next() {
Some('C') => 0,
@ -151,3 +170,18 @@ fn chunks<'a>(dump: &'a str) -> impl Iterator<Item = Cow<'a, str>> {
})
})
}
fn remove_skipped_item_from_beginning(s: &str) -> Option<&str> {
// Skip commented line
if let Some(after) = s.strip_prefix("--") {
Some(after.split_once('\n').unwrap_or_default().1)
}
// Skip view definition that's replaced later (the first definition selects all nulls)
else if let Some(after) = s.strip_prefix("CREATE VIEW ") {
let (name, after_name) = after.split_once(' ').unwrap_or_default();
Some(after_name.split_once("\n\n").unwrap_or_default().1)
.filter(|after_view| after_view.contains(&format!("\nCREATE OR REPLACE VIEW {name} ")))
} else {
None
}
}

4
migrations/2020-04-07-135912_add_user_community_apub_constraints/down.sql
View file

@ -2,9 +2,11 @@
DROP VIEW user_view CASCADE;
ALTER TABLE user_
ADD COLUMN fedi_name varchar(40) NOT NULL;
ADD COLUMN fedi_name varchar(40) NOT NULL DEFAULT 'http://fake.com';
ALTER TABLE user_
-- Default is only for existing rows
ALTER COLUMN fedi_name DROP DEFAULT,
ADD CONSTRAINT user__name_fedi_name_key UNIQUE (name, fedi_name);
-- Community

42
migrations/2020-04-14-163701_update_views_for_activitypub/down.sql
View file

@ -61,7 +61,17 @@ DROP VIEW community_aggregates_view CASCADE;
CREATE VIEW community_aggregates_view AS
SELECT
c.*,
c.id,
c.name,
c.title,
c.description,
c.category_id,
c.creator_id,
c.removed,
c.published,
c.updated,
c.deleted,
c.nsfw,
(
SELECT
name
@ -277,8 +287,23 @@ DROP VIEW post_aggregates_view;
-- regen post view
CREATE VIEW post_aggregates_view AS
SELECT
p.*,
SELECT p.id,
p.name,
p.url,
p.body,
p.creator_id,
p.community_id,
p.removed,
p.locked,
p.published,
p.updated,
p.deleted,
p.nsfw,
p.stickied,
p.embed_title,
p.embed_description,
p.embed_html,
p.thumbnail_url,
(
SELECT
u.banned
@ -511,7 +536,16 @@ DROP VIEW comment_aggregates_view;
-- reply and comment view
CREATE VIEW comment_aggregates_view AS
SELECT
c.*,
c.id,
c.creator_id,
c.post_id,
c.parent_id,
c.content,
c.removed,
c.read,
c.published,
c.updated,
c.deleted,
(
SELECT
community_id

249
migrations/2020-06-30-135809_remove_mat_views/down.sql
View file

@ -966,3 +966,252 @@ FROM
all_comment ac
LEFT JOIN user_mention um ON um.comment_id = ac.id;
-- comment
CREATE OR REPLACE FUNCTION refresh_comment ()
RETURNS TRIGGER
LANGUAGE plpgsql
AS $$
BEGIN
REFRESH MATERIALIZED VIEW CONCURRENTLY post_aggregates_mview;
REFRESH MATERIALIZED VIEW CONCURRENTLY comment_aggregates_mview;
REFRESH MATERIALIZED VIEW CONCURRENTLY community_aggregates_mview;
REFRESH MATERIALIZED VIEW CONCURRENTLY user_mview;
RETURN NULL;
END
$$;
CREATE OR REPLACE TRIGGER refresh_comment
AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON comment
FOR EACH statement
EXECUTE PROCEDURE refresh_comment ();
-- comment_like
CREATE OR REPLACE FUNCTION refresh_comment_like ()
RETURNS TRIGGER
LANGUAGE plpgsql
AS $$
BEGIN
REFRESH MATERIALIZED VIEW CONCURRENTLY comment_aggregates_mview;
REFRESH MATERIALIZED VIEW CONCURRENTLY user_mview;
RETURN NULL;
END
$$;
CREATE OR REPLACE TRIGGER refresh_comment_like
AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON comment_like
FOR EACH statement
EXECUTE PROCEDURE refresh_comment_like ();
-- community_follower
CREATE OR REPLACE FUNCTION refresh_community_follower ()
RETURNS TRIGGER
LANGUAGE plpgsql
AS $$
BEGIN
REFRESH MATERIALIZED VIEW CONCURRENTLY community_aggregates_mview;
REFRESH MATERIALIZED VIEW CONCURRENTLY post_aggregates_mview;
RETURN NULL;
END
$$;
CREATE or replace TRIGGER refresh_community_follower
AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON community_follower
FOR EACH statement
EXECUTE PROCEDURE refresh_community_follower ();
-- community_user_ban
CREATE OR REPLACE FUNCTION refresh_community_user_ban ()
RETURNS TRIGGER
LANGUAGE plpgsql
AS $$
BEGIN
REFRESH MATERIALIZED VIEW CONCURRENTLY comment_aggregates_mview;
REFRESH MATERIALIZED VIEW CONCURRENTLY post_aggregates_mview;
RETURN NULL;
END
$$;
CREATE or replace TRIGGER refresh_community_user_ban
AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON community_user_ban
FOR EACH statement
EXECUTE PROCEDURE refresh_community_user_ban ();
-- post_like
CREATE OR REPLACE FUNCTION refresh_post_like ()
RETURNS TRIGGER
LANGUAGE plpgsql
AS $$
BEGIN
REFRESH MATERIALIZED VIEW CONCURRENTLY post_aggregates_mview;
REFRESH MATERIALIZED VIEW CONCURRENTLY user_mview;
RETURN NULL;
END
$$;
CREATE or replace TRIGGER refresh_post_like
AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON post_like
FOR EACH statement
EXECUTE PROCEDURE refresh_post_like ();
CREATE or replace VIEW community_moderator_view AS
SELECT
*,
(
SELECT
actor_id
FROM
user_ u
WHERE
cm.user_id = u.id) AS user_actor_id,
(
SELECT
local
FROM
user_ u
WHERE
cm.user_id = u.id) AS user_local,
(
SELECT
name
FROM
user_ u
WHERE
cm.user_id = u.id) AS user_name,
(
SELECT
avatar
FROM
user_ u
WHERE
cm.user_id = u.id), (
SELECT
actor_id
FROM
community c
WHERE
cm.community_id = c.id) AS community_actor_id,
(
SELECT
local
FROM
community c
WHERE
cm.community_id = c.id) AS community_local,
(
SELECT
name
FROM
community c
WHERE
cm.community_id = c.id) AS community_name
FROM
community_moderator cm;
CREATE or replace VIEW community_follower_view AS
SELECT
*,
(
SELECT
actor_id
FROM
user_ u
WHERE
cf.user_id = u.id) AS user_actor_id,
(
SELECT
local
FROM
user_ u
WHERE
cf.user_id = u.id) AS user_local,
(
SELECT
name
FROM
user_ u
WHERE
cf.user_id = u.id) AS user_name,
(
SELECT
avatar
FROM
user_ u
WHERE
cf.user_id = u.id), (
SELECT
actor_id
FROM
community c
WHERE
cf.community_id = c.id) AS community_actor_id,
(
SELECT
local
FROM
community c
WHERE
cf.community_id = c.id) AS community_local,
(
SELECT
name
FROM
community c
WHERE
cf.community_id = c.id) AS community_name
FROM
community_follower cf;
CREATE or replace VIEW community_user_ban_view AS
SELECT
*,
(
SELECT
actor_id
FROM
user_ u
WHERE
cm.user_id = u.id) AS user_actor_id,
(
SELECT
local
FROM
user_ u
WHERE
cm.user_id = u.id) AS user_local,
(
SELECT
name
FROM
user_ u
WHERE
cm.user_id = u.id) AS user_name,
(
SELECT
avatar
FROM
user_ u
WHERE
cm.user_id = u.id), (
SELECT
actor_id
FROM
community c
WHERE
cm.community_id = c.id) AS community_actor_id,
(
SELECT
local
FROM
community c
WHERE
cm.community_id = c.id) AS community_local,
(
SELECT
name
FROM
community c
WHERE
cm.community_id = c.id) AS community_name
FROM
community_user_ban cm;

2
migrations/2020-07-08-202609_add_creator_published/down.sql
View file

@ -493,3 +493,5 @@ SELECT
FROM
post_aggregates_fast pav;
CREATE INDEX idx_post_aggregates_fast_hot_rank_published ON post_aggregates_fast (hot_rank DESC, published DESC);

31
migrations/2020-10-07-234221_fix_fast_triggers/down.sql
View file

@ -155,7 +155,8 @@ BEGIN
UPDATE
post_aggregates_fast AS paf
SET
hot_rank = pav.hot_rank
hot_rank = pav.hot_rank,
hot_rank_active = pav.hot_rank_active
FROM
post_aggregates_view AS pav
WHERE
@ -220,14 +221,36 @@ BEGIN
post_aggregates_view
WHERE
id = NEW.post_id;
-- Force the hot rank as zero on week-older posts
-- Update the comment hot_ranks as of last week
UPDATE
comment_aggregates_fast AS caf
SET
hot_rank = cav.hot_rank,
hot_rank_active = cav.hot_rank_active
FROM
comment_aggregates_view AS cav
WHERE
caf.id = cav.id
AND (cav.published > ('now'::timestamp - '1 week'::interval));
-- Update the post ranks
UPDATE
post_aggregates_fast AS paf
SET
hot_rank = 0
hot_rank = pav.hot_rank,
hot_rank_active = pav.hot_rank_active
FROM
post_aggregates_view AS pav
WHERE
paf.id = pav.id
AND (pav.published > ('now'::timestamp - '1 week'::interval));
-- Force the hot rank active as zero on 2 day-older posts (necro-bump)
UPDATE
post_aggregates_fast AS paf
SET
hot_rank_active = 0
WHERE
paf.id = NEW.post_id
AND (paf.published < ('now'::timestamp - '1 week'::interval));
AND (paf.published < ('now'::timestamp - '2 days'::interval));
-- Update community number of comments
UPDATE
community_aggregates_fast AS caf