diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md new file mode 100644 index 00000000..b57e0e12 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md @@ -0,0 +1,27 @@ +--- +name: "\U0001F41E Bug Report" +about: Create a report to help us improve Lemmy +title: '' +labels: bug +assignees: '' + +--- + +Found a bug? Please fill out the sections below. 👍 + +### Issue Summary + +A summary of the bug. + + +### Steps to Reproduce + +1. (for example) I clicked login, and an endless spinner show up. +2. I tried to install lemmy via this guide, and I'm getting this error. +3. ... + +### Technical details + +* Please post your log: `sudo docker-compose logs > lemmy_log.out`. +* What OS are you trying to install lemmy on? +* Any browser console errors? diff --git a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md new file mode 100644 index 00000000..957f4cdf --- /dev/null +++ b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md @@ -0,0 +1,42 @@ +--- +name: "\U0001F680 Feature request" +about: Suggest an idea for improving Lemmy +title: '' +labels: enhancement +assignees: '' + +--- + +### Is your proposal related to a problem? + + + +(Write your answer here.) + +### Describe the solution you'd like + + + +(Describe your proposed solution here.) + +### Describe alternatives you've considered + + + +(Write your answer here.) + +### Additional context + + + +(Write your answer here.) diff --git a/.github/ISSUE_TEMPLATE/QUESTION.md b/.github/ISSUE_TEMPLATE/QUESTION.md new file mode 100644 index 00000000..b45f8f1e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/QUESTION.md @@ -0,0 +1,10 @@ +--- +name: "? Question" +about: General questions about Lemmy +title: '' +labels: question +assignees: '' + +--- + +What's the question you have about lemmy? diff --git a/.travis.yml b/.travis.yml index 602a8613..9541afaa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,10 +24,11 @@ script: - cargo clippy -- -D clippy::style -D clippy::correctness -D clippy::complexity -D clippy::perf - cargo install diesel_cli --no-default-features --features postgres --force - diesel migration run - - cargo test + - cargo test --workspace env: global: - DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy + - LEMMY_DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy - RUST_TEST_THREADS=1 addons: diff --git a/ansible/VERSION b/ansible/VERSION index 151a6866..935ddfd2 100644 --- a/ansible/VERSION +++ b/ansible/VERSION @@ -1 +1 @@ -v0.7.8 +v0.7.16 diff --git a/ansible/lemmy.yml b/ansible/lemmy.yml index 5c8a5f91..3520c404 100644 --- a/ansible/lemmy.yml +++ b/ansible/lemmy.yml @@ -11,6 +11,7 @@ when: lemmy_base_dir is not defined - name: install python for Ansible + # python2-minimal instead of python-minimal for ubuntu 20.04 and up raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal python-setuptools) args: executable: /bin/bash @@ -27,7 +28,18 @@ - 'docker-compose' - 'docker.io' - 'certbot' + + - name: install certbot-nginx on ubuntu < 20 + apt: + pkg: - 'python-certbot-nginx' + when: ansible_distribution == 'Ubuntu' and ansible_distribution_version is version('20.04', '<') + + - name: install certbot-nginx on ubuntu > 20 + apt: + pkg: + - 'python3-certbot-nginx' + when: ansible_distribution == 'Ubuntu' and ansible_distribution_version is version('20.04', '>=') - name: request initial letsencrypt certificate command: certbot certonly --nginx --agree-tos -d '{{ domain }}' -m '{{ letsencrypt_contact_email }}' diff --git a/ansible/templates/docker-compose.yml b/ansible/templates/docker-compose.yml index 76c53463..f4c94fd7 100644 --- a/ansible/templates/docker-compose.yml +++ b/ansible/templates/docker-compose.yml @@ -35,7 +35,7 @@ services: restart: always iframely: - image: jolt/iframely:v1.4.3 + image: dogbin/iframely:latest ports: - "127.0.0.1:8061:80" volumes: diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index 82a03f3c..4445e4fe 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -17,13 +17,20 @@ WORKDIR /app RUN sudo chown -R rust:rust . RUN USER=root cargo new server WORKDIR /app/server +RUN mkdir -p lemmy_db/src/ lemmy_utils/src/ COPY server/Cargo.toml server/Cargo.lock ./ +COPY server/lemmy_db/Cargo.toml ./lemmy_db/ +COPY server/lemmy_utils/Cargo.toml ./lemmy_utils/ RUN sudo chown -R rust:rust . RUN mkdir -p ./src/bin \ - && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs + && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs \ + && cp ./src/bin/main.rs ./lemmy_db/src/main.rs \ + && cp ./src/bin/main.rs ./lemmy_utils/src/main.rs RUN cargo build RUN rm -f ./target/x86_64-unknown-linux-musl/release/deps/lemmy_server* COPY server/src ./src/ +COPY server/lemmy_db ./lemmy_db/ +COPY server/lemmy_utils ./lemmy_utils/ COPY server/migrations ./migrations/ # Build for debug diff --git a/docker/dev/docker-compose.yml b/docker/dev/docker-compose.yml index fc6e2105..51a3ecda 100644 --- a/docker/dev/docker-compose.yml +++ b/docker/dev/docker-compose.yml @@ -40,7 +40,7 @@ services: restart: always iframely: - image: jolt/iframely:v1.4.3 + image: dogbin/iframely:latest ports: - "127.0.0.1:8061:80" volumes: diff --git a/docker/federation-test/run-tests.sh b/docker/federation-test/run-tests.sh index 57c6cc8f..3848414b 100755 --- a/docker/federation-test/run-tests.sh +++ b/docker/federation-test/run-tests.sh @@ -1,6 +1,10 @@ #!/bin/bash set -e +# make sure there are no old containers or old data around +sudo docker-compose --file ../federation/docker-compose.yml --project-directory . down +sudo rm -rf volumes + pushd ../../server/ cargo build popd diff --git a/docker/federation/docker-compose.yml b/docker/federation/docker-compose.yml index 51a37b91..c552d18f 100644 --- a/docker/federation/docker-compose.yml +++ b/docker/federation/docker-compose.yml @@ -107,6 +107,7 @@ services: - ./volumes/postgres_gamma:/var/lib/postgresql/data iframely: - image: jolt/iframely:v1.4.3 + image: dogbin/iframely:latest volumes: - ../iframely.config.local.js:/iframely/config.local.js:ro + restart: always diff --git a/docker/prod/Dockerfile b/docker/prod/Dockerfile index 54485a37..9000ca3a 100644 --- a/docker/prod/Dockerfile +++ b/docker/prod/Dockerfile @@ -10,13 +10,19 @@ WORKDIR /app RUN sudo chown -R rust:rust . RUN USER=root cargo new server WORKDIR /app/server +RUN mkdir -p lemmy_db/src/ lemmy_utils/src/ COPY --chown=rust:rust server/Cargo.toml server/Cargo.lock ./ -#RUN sudo chown -R rust:rust . +COPY --chown=rust:rust server/lemmy_db/Cargo.toml ./lemmy_db/ +COPY --chown=rust:rust server/lemmy_utils/Cargo.toml ./lemmy_utils/ RUN mkdir -p ./src/bin \ - && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs + && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs \ + && cp ./src/bin/main.rs ./lemmy_db/src/main.rs \ + && cp ./src/bin/main.rs ./lemmy_utils/src/main.rs RUN cargo build --release RUN rm -f ./target/$CARGO_BUILD_TARGET/$RUSTRELEASEDIR/deps/lemmy_server* COPY --chown=rust:rust server/src ./src/ +COPY --chown=rust:rust server/lemmy_db ./lemmy_db/ +COPY --chown=rust:rust server/lemmy_utils ./lemmy_utils/ COPY --chown=rust:rust server/migrations ./migrations/ # build for release diff --git a/docker/prod/docker-compose.yml b/docker/prod/docker-compose.yml index 5ccb1ec5..6c3bccb0 100644 --- a/docker/prod/docker-compose.yml +++ b/docker/prod/docker-compose.yml @@ -12,7 +12,7 @@ services: restart: always lemmy: - image: dessalines/lemmy:v0.7.8 + image: dessalines/lemmy:v0.7.16 ports: - "127.0.0.1:8536:8536" restart: always @@ -35,7 +35,7 @@ services: restart: always iframely: - image: jolt/iframely:v1.4.3 + image: dogbin/iframely:latest ports: - "127.0.0.1:8061:80" volumes: diff --git a/docker/prod/migrate-pictshare-to-pictrs.bash b/docker/prod/migrate-pictshare-to-pictrs.bash index 8229eb28..667183a0 100644 --- a/docker/prod/migrate-pictshare-to-pictrs.bash +++ b/docker/prod/migrate-pictshare-to-pictrs.bash @@ -1,16 +1,21 @@ #!/bin/bash set -e -if [[ $(id -u) != 0 ]]; then +if [[ $(id -u) != 0 ]]; then echo "This migration needs to be run as root" exit fi -if [[ ! -f docker-compose.yml ]]; then +if [[ ! -f docker-compose.yml ]]; then echo "No docker-compose.yml found in current directory. Is this the right folder?" exit fi +if ! which jq > /dev/null; then + echo "jq must be installed to run this migration. On ubuntu systems, try 'sudo apt-get install jq'" + exit +fi + # Fixing pictrs permissions mkdir -p volumes/pictrs sudo chown -R 991:991 volumes/pictrs @@ -26,6 +31,8 @@ fi # echo "Stopping Lemmy so that users dont upload new images during the migration" # docker-compose stop lemmy +CRASHED_ON=() + pushd volumes/pictshare/ echo "Importing pictshare images to pict-rs..." IMAGE_NAMES=* @@ -34,11 +41,36 @@ for image in $IMAGE_NAMES; do if [[ ! -f $IMAGE_PATH ]]; then continue fi - echo -e "\nImporting $IMAGE_PATH" - ret=0 - curl --silent --fail -F "images[]=@$IMAGE_PATH" http://127.0.0.1:8537/import || ret=$? - if [[ $ret != 0 ]]; then - echo "Error for $IMAGE_PATH : $ret" + res=$(curl -s -F "images[]=@$IMAGE_PATH" http://127.0.0.1:8537/import | jq .msg) + if [ "${res}" == "" ]; then + echo -n "C" >&2 + echo "" + CRASHED_ON+=("${IMAGE_PATH}") + echo "Failed to import $IMAGE_PATH with no error message" + echo " assuming crash, sleeping" + sleep 10 + continue + fi + if [ "${res}" != "\"ok\"" ]; then + echo -n "F" >&2 + echo "" + echo "Failed to import $IMAGE_PATH" + echo " Reason: ${res}" + else + echo -n "." >&2 + fi +done + +for image in ${CRASHED_ON[@]}; do + echo "Retrying ${image}" + res=$(curl -s -F "images[]=@$IMAGE_PATH" http://127.0.0.1:8537/import | jq .msg) + if [ "${res}" != "\"ok\"" ]; then + echo -n "F" >&2 + echo "" + echo "Failed to upload ${image} on 2nd attempt" + echo " Reason: ${res}" + else + echo -n "." >&2 fi done diff --git a/docs/src/administration_configuration.md b/docs/src/administration_configuration.md index cc421b0b..56448de4 100644 --- a/docs/src/administration_configuration.md +++ b/docs/src/administration_configuration.md @@ -7,7 +7,7 @@ can copy the options you want to change into your local `config.hjson` file. Additionally, you can override any config files with environment variables. These have the same name as the config options, and are prefixed with `LEMMY_`. For example, you can override the -`database.password` with `LEMMY__DATABASE__POOL_SIZE=10`. +`database.password` with `LEMMY_DATABASE__POOL_SIZE=10`. An additional option `LEMMY_DATABASE_URL` is available, which can be used with a PostgreSQL connection string like `postgres://lemmy:password@lemmy_db:5432/lemmy`, passing all connection diff --git a/docs/src/contributing_federation_development.md b/docs/src/contributing_federation_development.md index 520a6127..143ae9f8 100644 --- a/docs/src/contributing_federation_development.md +++ b/docs/src/contributing_federation_development.md @@ -5,14 +5,7 @@ If you don't have a local clone of the Lemmy repo yet, just run the following command: ```bash -git clone https://github.com/LemmyNet/lemmy -b federation -``` - -If you already have the Lemmy repo cloned, you need to add a new remote: -```bash -git remote add federation https://github.com/LemmyNet/lemmy -git checkout federation -git pull federation federation +git clone https://github.com/LemmyNet/lemmy ``` ## Running locally @@ -26,18 +19,34 @@ You need to have the following packages installed, the Docker service needs to b Then run the following ```bash -cd dev/federation-test -./run-federation-test.bash +cd docker/federation +./run-federation-test.bash -yarn ``` -After the build is finished and the docker-compose setup is running, open [127.0.0.1:8540](http://127.0.0.1:8540) and -[127.0.0.1:8550](http://127.0.0.1:8550) in your browser to use the test instances. You can login as admin with -username `lemmy_alpha` and `lemmy_beta` respectively, with password `lemmy`. +The federation test sets up 3 instances: + +Instance / Username | Location +--- | --- +lemmy_alpha | [127.0.0.1:8540](http://127.0.0.1:8540) +lemmy_beta | [127.0.0.1:8550](http://127.0.0.1:8550) +lemmy_gamma | [127.0.0.1:8560](http://127.0.0.1:8560) + +You can log into each using the instance name, and `lemmy` as the password, IE (`lemmy_alpha`, `lemmy`). + +Firefox containers are a good way to test them interacting. + +## Integration tests + +To run a suite of suite of federation integration tests: + +```bash +cd docker/federation-test +./run-tests.sh +``` ## Running on a server -Note that federation is currently in alpha. Only use it for testing, not on any production server, and be aware -that you might have to wipe the instance data at one point or another. +Note that federation is currently in alpha. **Only use it for testing**, not on any production server, and be aware that turning on federation may break your instance. Follow the normal installation instructions, either with [Ansible](administration_install_ansible.md) or [manually](administration_install_docker.md). Then replace the line `image: dessalines/lemmy:v0.x.x` in @@ -47,11 +56,12 @@ Follow the normal installation instructions, either with [Ansible](administratio ``` federation: { enabled: true - allowed_instances: example.com + tls_enabled: true, + allowed_instances: example.com, } ``` -Afterwards, and whenver you want to update to the latest version, run these commands on the server: +Afterwards, and whenever you want to update to the latest version, run these commands on the server: ``` cd /lemmy/ diff --git a/docs/src/contributing_tests.md b/docs/src/contributing_tests.md index 13e5d122..d4168e19 100644 --- a/docs/src/contributing_tests.md +++ b/docs/src/contributing_tests.md @@ -7,9 +7,7 @@ following commands in the `server` subfolder: ```bash psql -U lemmy -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;" -export DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy -diesel migration run -RUST_TEST_THREADS=1 cargo test +./test.sh ``` ### Federation diff --git a/docs/src/lemmy_council.md b/docs/src/lemmy_council.md index 9b24522a..d5b9c790 100644 --- a/docs/src/lemmy_council.md +++ b/docs/src/lemmy_council.md @@ -3,30 +3,52 @@ - A group of lemmy developers and users that use a well-defined democratic process to steer the project in a positive direction, keep it aligned to community goals, and resolve conflicts. - Council members are also added as administrators to any official Lemmy instances. -## Voting / Decision-Making +## 1. What gets voted on -### Process -- Anything is open for discussion -- Voting done through matrix chat reacts (thumbs up/thumbs down) -- Require a simple majority for votes. (Maybe 2/3rds for more debated decisions). -- Once a decision is reached democratically, the dicision is binding and all group members have to follow it -- All members of the Lemmy council have equal voting power. -- Voting must stay open for at least 2 days. +This section describes all the aspects of Lemmy where the council has decision making power, namely: -### What gets voted on -- Membership (joining, removing) - Coding direction - Priorities / Emphasis - Controversial features (For example, an unpopular feature should be removed) -- Communication mediums -- Conflict resolution -- dev.lemmy.ml (domain and server) -- lemmy.ml and subdomains (excluding communism.lemmy.ml) -- git repo including mirrors (on github, gitea, etc) -- Any official accounts of the Lemmy project, for example the Mastodon account or the Liberapay account +- Moderation and conflict resolution on: + - [dev.lemmy.ml](https://dev.lemmy.ml/) + - [github.com/LemmyNet/lemmy](https://github.com/LemmyNet/lemmy) + - [yerbamate.dev/LemmyNet/lemmy](https://yerbamate.dev/LemmyNet/lemmy) + - [weblate.yerbamate.dev/projects/lemmy/](https://weblate.yerbamate.dev/projects/lemmy/) +- Technical administration of dev.lemmy.ml +- Official Lemmy accounts + - [Mastodon](https://mastodon.social/@LemmyDev) + - [Liberapay](https://liberapay.com/Lemmy/) + - [Patreon](https://www.patreon.com/dessalines) +- Council membership changes - Changes to these rules -## Joining +## 2. Feedback and Activity Reports + +Every week, the council should make a thread on Lemmy that details its activity during the past week, be it development, moderation, or anything else mentioned in 1. + +At the same time, users can give feedback and suggestions in this thread. This should be taken into account by the council. Council members can call for a vote on any controversial issues, if they can't be resolved by discussion. + +## 2. Voting Process + +Most of the time, we keep each other up to date through the Matrix chat, and take informal decisions on uncontroversial issues. For example, a user clearly violating the site rules could be banned by a single person, or ideally after discussing it with at least one other member. + +If an issue can not be resolved in this way, then any council member can call for a vote, which works in the following way: + +- Any council member can call for a vote, on any topic mentioned in 1. +- This should be used if there is any controversy in the community, or between council members. +- Before taking any decision, there needs to be a discussion where every council member can +explain their position. +- Discussion should be taken with the goal of reaching a compromise that is acceptable for +everyone. +- After the discussion, voting is done through Matrix emojis (👍: yes, 👎: no, X: abstain) and must +stay open for at least two days. +- All members of the Lemmy council have equal voting power. +- Decisions should be reached unanimously, or nearly so. If this is not possible, at least +2/3 of votes must be in favour for the motion to pass. +- Once a decision is reached in this way, every member needs to abide by it. + +## 4. Joining - We use the following process: anyone who is active around Lemmy can recommend any other active person to join the council. This has to be approved by a majority of the council. - Active users are defined as those who contribute to Lemmy in some way for at least an hour per week on average, doing things like reporting bugs, discussing rules and features, translating, promoting, developing, or doing other things that aim to improve Lemmy as a whole. -> people should have joined at least a month ago. @@ -34,23 +56,24 @@ - Note: we would like to have a process where community members can elect candidates for the council, but this is not realistic because a single user could easily create multiple accounts and cheat the vote. - Limit growth to one new member per month at most. -## Removing members +## 5. Removing members - Inactive members should be removed from the council after a few months of inactivity, and after receiving a notification about this. - Members that dont follow binding council decisions should be removed. - Any member can be removed in a vote. -## Goals +## 6. Goals - We encourage the membership of groups such as LGBT, religious or ethnic minorities, abuse victims, etc etc, and strive to create a safe space for them to express their opinions. We also support measures to increase participation by the previously mentioned groups. - The following are banned, and will always be harshly punished: fascism, abuse, racism, sexism, etc etc, -## Communication +## 7. Communication - A private Matrix chat for all council members. - (Once private communities are done) A private community on dev.lemmy.ml for issues. -## Member List / Contact Info +## 8. Member List / Contact Info General Contact [@LemmyDev Mastodon](https://mastodon.social/@LemmyDev) - [Dessalines](https://dev.lemmy.ml/u/dessalines) - [Nutomic](https://dev.lemmy.ml/u/nutomic) - [AgreeableLandscape](https://dev.lemmy.ml/u/AgreeableLandscape) - [fruechtchen](https://dev.lemmy.ml/u/fruechtchen) +- [kixiQu](https://dev.lemmy.ml/u/kixiQu) diff --git a/install.sh b/install.sh index fb42b26d..19b847b1 100755 --- a/install.sh +++ b/install.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -e # Set the database variable to the default first. diff --git a/server/Cargo.lock b/server/Cargo.lock index c54419e1..62438593 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -63,7 +63,7 @@ dependencies = [ "futures-util", "log", "once_cell", - "parking_lot", + "parking_lot 0.10.2", "pin-project", "smallvec", "tokio", @@ -269,7 +269,7 @@ dependencies = [ "lazy_static", "log", "num_cpus", - "parking_lot", + "parking_lot 0.10.2", "threadpool", ] @@ -397,6 +397,12 @@ dependencies = [ "gimli", ] +[[package]] +name = "adler" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc9a9dd069569f212bc4330af9f17c4afb5e8ce185e83dbb14f1349dda18b10" + [[package]] name = "adler32" version = "1.1.0" @@ -506,7 +512,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.3.7", "object", "rustc-demangle", ] @@ -680,9 +686,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.55" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1be3409f94d7bdceeb5f5fac551039d9b3f00e25da7a74fc4d33400a0d96368" +checksum = "0fde55d2a2bfaa4c9668bbc63f531fbdeee3ffe188f4662511ce2c22b3eedebe" [[package]] name = "cfg-if" @@ -692,9 +698,9 @@ checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" [[package]] name = "chrono" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80094f509cf8b5ae86a4966a39b3ff66cd7e2a3e594accec3743ff3fabeab5b2" +checksum = "f0fee792e164f78f5fe0c296cc2eb3688a2ca2b70cdff33040922d298203f0c4" dependencies = [ "num-integer", "num-traits 0.2.12", @@ -726,6 +732,15 @@ dependencies = [ "bitflags", ] +[[package]] +name = "cloudabi" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467" +dependencies = [ + "bitflags", +] + [[package]] name = "comrak" version = "0.7.0" @@ -1120,14 +1135,14 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42" +checksum = "68c90b0fc46cf89d227cc78b40e494ff81287a92dd07631e5af0d06fe3cf885e" dependencies = [ "cfg-if", "crc32fast", "libc", - "miniz_oxide", + "miniz_oxide 0.4.0", ] [[package]] @@ -1384,12 +1399,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "htmlescape" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" - [[package]] name = "http" version = "0.2.1" @@ -1470,6 +1479,12 @@ dependencies = [ "autocfg 1.0.0", ] +[[package]] +name = "instant" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69da7ce1490173c2bf4d26bc8be429aaeeaf4cce6c4b970b7949651fa17655fe" + [[package]] name = "iovec" version = "0.1.4" @@ -1508,18 +1523,18 @@ checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" [[package]] name = "js-sys" -version = "0.3.40" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce10c23ad2ea25ceca0093bd3192229da4c5b3c0f2de499c1ecac0d98d452177" +checksum = "c4b9172132a62451e56142bff9afc91c8e4a4500aa5b847da36815b63bfda916" dependencies = [ "wasm-bindgen", ] [[package]] name = "jsonwebtoken" -version = "7.1.2" +version = "7.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f325ae57ddcf609f02d891486ce740f5bbd0cc3e93f9bffaacdf6594b21404" +checksum = "afabcc15e437a6484fc4f12d0fd63068fe457bf93f1c148d3d9649c60b103f32" dependencies = [ "base64 0.12.3", "pem", @@ -1551,6 +1566,21 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lemmy_db" +version = "0.1.0" +dependencies = [ + "bcrypt", + "chrono", + "diesel", + "log", + "serde 1.0.114", + "serde_json", + "sha2", + "strum", + "strum_macros", +] + [[package]] name = "lemmy_server" version = "0.0.1" @@ -1568,27 +1598,23 @@ dependencies = [ "base64 0.12.3", "bcrypt", "chrono", - "comrak", - "config", "diesel", "diesel_migrations", "dotenv", "env_logger", "failure", "futures", - "htmlescape", "http", "http-signature-normalization-actix", "itertools", "jsonwebtoken", "lazy_static", - "lettre", - "lettre_email", + "lemmy_db", + "lemmy_utils", "log", "openssl", "percent-encoding", "rand 0.7.3", - "regex", "rss", "serde 1.0.114", "serde_json", @@ -1600,6 +1626,26 @@ dependencies = [ "uuid 0.8.1", ] +[[package]] +name = "lemmy_utils" +version = "0.1.0" +dependencies = [ + "chrono", + "comrak", + "config", + "itertools", + "lazy_static", + "lettre", + "lettre_email", + "log", + "openssl", + "rand 0.7.3", + "regex", + "serde 1.0.114", + "serde_json", + "url", +] + [[package]] name = "lettre" version = "0.9.3" @@ -1676,6 +1722,15 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lock_api" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de302ce1fe7482db13738fbaf2e21cfb06a986b89c0bf38d88abf16681aada4e" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.8" @@ -1781,6 +1836,15 @@ dependencies = [ "adler32", ] +[[package]] +name = "miniz_oxide" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0f75932c1f6cfae3c04000e40114adf955636e19040f9c0a2c380702aa1c7f" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.6.22" @@ -1985,8 +2049,19 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" dependencies = [ - "lock_api", - "parking_lot_core", + "lock_api 0.3.4", + "parking_lot_core 0.7.2", +] + +[[package]] +name = "parking_lot" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4893845fa2ca272e647da5d0e46660a314ead9c2fdd9a883aabc32e481a8733" +dependencies = [ + "instant", + "lock_api 0.4.0", + "parking_lot_core 0.8.0", ] [[package]] @@ -1996,7 +2071,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" dependencies = [ "cfg-if", - "cloudabi", + "cloudabi 0.0.3", + "libc", + "redox_syscall", + "smallvec", + "winapi 0.3.9", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" +dependencies = [ + "cfg-if", + "cloudabi 0.1.0", + "instant", "libc", "redox_syscall", "smallvec", @@ -2164,12 +2254,12 @@ dependencies = [ [[package]] name = "r2d2" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1497e40855348e4a8a40767d8e55174bce1e445a3ac9254ad44ad468ee0485af" +checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" dependencies = [ "log", - "parking_lot", + "parking_lot 0.11.0", "scheduled-thread-pool", ] @@ -2306,7 +2396,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" dependencies = [ - "cloudabi", + "cloudabi 0.0.3", "fuchsia-cprng", "libc", "rand_core 0.4.2", @@ -2462,11 +2552,11 @@ dependencies = [ [[package]] name = "scheduled-thread-pool" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0988d7fdf88d5e5fcf5923a0f1e8ab345f3e98ab4bc6bc45a2d5ff7f7458fbf6" +checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" dependencies = [ - "parking_lot", + "parking_lot 0.11.0", ] [[package]] @@ -2570,9 +2660,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.55" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec2c5d7e739bc07a3e73381a39d61fdb5f671c60c1df26a130690665803d8226" +checksum = "3433e879a558dde8b5e8feb2a04899cf34fdde1fafb894687e52105fc1162ac3" dependencies = [ "indexmap", "itoa", @@ -3106,9 +3196,9 @@ checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" [[package]] name = "unicode-width" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" [[package]] name = "unicode-xid" @@ -3222,9 +3312,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasm-bindgen" -version = "0.2.63" +version = "0.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c2dc4aa152834bc334f506c1a06b866416a8b6697d5c9f75b9a689c8486def0" +checksum = "6a634620115e4a229108b71bde263bb4220c483b3f07f5ba514ee8d15064c4c2" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3232,9 +3322,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.63" +version = "0.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded84f06e0ed21499f6184df0e0cb3494727b0c5da89534e0fcc55c51d812101" +checksum = "3e53963b583d18a5aa3aaae4b4c1cb535218246131ba22a71f05b518098571df" dependencies = [ "bumpalo", "lazy_static", @@ -3247,9 +3337,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.63" +version = "0.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "838e423688dac18d73e31edce74ddfac468e37b1506ad163ffaf0a46f703ffe3" +checksum = "3fcfd5ef6eec85623b4c6e844293d4516470d8f19cd72d0d12246017eb9060b8" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3257,9 +3347,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.63" +version = "0.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3156052d8ec77142051a533cdd686cba889537b213f948cd1d20869926e68e92" +checksum = "9adff9ee0e94b926ca81b57f57f86d5545cdcb1d259e21ec9bdd95b901754c75" dependencies = [ "proc-macro2", "quote", @@ -3270,15 +3360,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.63" +version = "0.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9ba19973a58daf4db6f352eda73dc0e289493cd29fb2632eb172085b6521acd" +checksum = "7f7b90ea6c632dd06fd765d44542e234d5e63d9bb917ecd64d79778a13bd79ae" [[package]] name = "web-sys" -version = "0.3.40" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b72fe77fd39e4bd3eaa4412fd299a0be6b3dfe9d2597e2f1c20beb968f41d17" +checksum = "863539788676619aac1a23e2df3655e96b32b0e05eb72ca34ba045ad573c625d" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/server/Cargo.toml b/server/Cargo.toml index 8daf72c4..a5e5a583 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,14 +1,21 @@ [package] name = "lemmy_server" version = "0.0.1" -authors = ["Dessalines "] edition = "2018" [profile.release] lto = true +[workspace] +members = [ + "lemmy_utils", + "lemmy_db" +] + [dependencies] -diesel = { version = "1.4.4", features = ["postgres","chrono","r2d2","64-column-tables","serde_json"] } +lemmy_utils = { path = "./lemmy_utils" } +lemmy_db = { path = "./lemmy_db" } +diesel = "1.4.4" diesel_migrations = "1.4.0" dotenv = "0.15.0" activitystreams = "0.6.2" @@ -31,16 +38,10 @@ rand = "0.7.3" strum = "0.18.0" strum_macros = "0.18.0" jsonwebtoken = "7.0.1" -regex = "1.3.5" lazy_static = "1.3.0" -lettre = "0.9.3" -lettre_email = "0.9.4" rss = "1.9.0" -htmlescape = "0.3.1" url = { version = "2.1.1", features = ["serde"] } -config = {version = "0.10.1", default-features = false, features = ["hjson"] } percent-encoding = "2.1.0" -comrak = "0.7" openssl = "0.10" http = "0.2.1" http-signature-normalization-actix = { version = "0.4.0-alpha.0", default-features = false, features = ["sha-2"] } diff --git a/server/db-init.sh b/server/db-init.sh index a2ad77b5..ccecb7de 100755 --- a/server/db-init.sh +++ b/server/db-init.sh @@ -1,4 +1,5 @@ -#!/bin/sh +#!/bin/bash +set -e # Default configurations username=lemmy diff --git a/server/diesel.toml b/server/diesel.toml index 92267c82..1644558f 100644 --- a/server/diesel.toml +++ b/server/diesel.toml @@ -2,4 +2,4 @@ # see diesel.rs/guides/configuring-diesel-cli [print_schema] -file = "src/schema.rs" +file = "lemmy_db/src/schema.rs" diff --git a/server/lemmy_db/Cargo.toml b/server/lemmy_db/Cargo.toml new file mode 100644 index 00000000..d94cf5fc --- /dev/null +++ b/server/lemmy_db/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "lemmy_db" +version = "0.1.0" +edition = "2018" + +[dependencies] +diesel = { version = "1.4.4", features = ["postgres","chrono","r2d2","64-column-tables","serde_json"] } +chrono = { version = "0.4.7", features = ["serde"] } +serde = { version = "1.0.105", features = ["derive"] } +serde_json = { version = "1.0.52", features = ["preserve_order"]} +strum = "0.18.0" +strum_macros = "0.18.0" +log = "0.4.0" +sha2 = "0.9" +bcrypt = "0.8.0" \ No newline at end of file diff --git a/server/src/db/activity.rs b/server/lemmy_db/src/activity.rs similarity index 84% rename from server/src/db/activity.rs rename to server/lemmy_db/src/activity.rs index 8c2b0c74..83f85ca1 100644 --- a/server/src/db/activity.rs +++ b/server/lemmy_db/src/activity.rs @@ -1,9 +1,12 @@ -use crate::{blocking, db::Crud, schema::activity, DbPool, LemmyError}; +use crate::{schema::activity, Crud}; use diesel::{dsl::*, result::Error, *}; use log::debug; use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::fmt::Debug; +use std::{ + fmt::Debug, + io::{Error as IoError, ErrorKind}, +}; #[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, Deserialize)] #[table_name = "activity"] @@ -55,46 +58,43 @@ impl Crud for Activity { } } -pub async fn insert_activity( - user_id: i32, - data: T, - local: bool, - pool: &DbPool, -) -> Result<(), LemmyError> -where - T: Serialize + Debug + Send + 'static, -{ - blocking(pool, move |conn| { - do_insert_activity(conn, user_id, &data, local) - }) - .await??; - Ok(()) -} - -fn do_insert_activity( +pub fn do_insert_activity( conn: &PgConnection, user_id: i32, data: &T, local: bool, -) -> Result<(), LemmyError> +) -> Result where T: Serialize + Debug, { + debug!("inserting activity for user {}, data {:?}", user_id, &data); let activity_form = ActivityForm { user_id, data: serde_json::to_value(&data)?, local, updated: None, }; - debug!("inserting activity for user {}, data {:?}", user_id, data); - Activity::create(&conn, &activity_form)?; - Ok(()) + let result = Activity::create(&conn, &activity_form); + match result { + Ok(s) => Ok(s), + Err(e) => Err(IoError::new( + ErrorKind::Other, + format!("Failed to insert activity into database: {}", e), + )), + } } #[cfg(test)] mod tests { - use super::{super::user::*, *}; - use crate::db::{establish_unpooled_connection, Crud, ListingType, SortType}; + use crate::{ + activity::{Activity, ActivityForm}, + tests::establish_unpooled_connection, + user::{UserForm, User_}, + Crud, + ListingType, + SortType, + }; + use serde_json::Value; #[test] fn test_crud() { diff --git a/server/src/db/category.rs b/server/lemmy_db/src/category.rs similarity index 95% rename from server/src/db/category.rs rename to server/lemmy_db/src/category.rs index ff49bbbe..ec2efc7b 100644 --- a/server/src/db/category.rs +++ b/server/lemmy_db/src/category.rs @@ -1,6 +1,6 @@ use crate::{ - db::Crud, schema::{category, category::dsl::*}, + Crud, }; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -52,8 +52,7 @@ impl Category { #[cfg(test)] mod tests { - use super::*; - use crate::db::establish_unpooled_connection; + use crate::{category::Category, tests::establish_unpooled_connection}; #[test] fn test_crud() { diff --git a/server/src/db/comment.rs b/server/lemmy_db/src/comment.rs similarity index 96% rename from server/src/db/comment.rs rename to server/lemmy_db/src/comment.rs index 7e76770f..602070d5 100644 --- a/server/src/db/comment.rs +++ b/server/lemmy_db/src/comment.rs @@ -1,9 +1,5 @@ use super::{post::Post, *}; -use crate::{ - apub::{make_apub_endpoint, EndpointType}, - naive_now, - schema::{comment, comment_like, comment_saved}, -}; +use crate::schema::{comment, comment_like, comment_saved}; // WITH RECURSIVE MyTree AS ( // SELECT * FROM comment WHERE parent_id IS NULL @@ -77,12 +73,15 @@ impl Crud for Comment { } impl Comment { - pub fn update_ap_id(conn: &PgConnection, comment_id: i32) -> Result { + pub fn update_ap_id( + conn: &PgConnection, + comment_id: i32, + apub_id: String, + ) -> Result { use crate::schema::comment::dsl::*; - let apid = make_apub_endpoint(EndpointType::Comment, &comment_id.to_string()).to_string(); diesel::update(comment.find(comment_id)) - .set(ap_id.eq(apid)) + .set(ap_id.eq(apub_id)) .get_result::(conn) } @@ -204,10 +203,8 @@ impl Saveable for CommentSaved { #[cfg(test)] mod tests { - use super::{ - super::{community::*, post::*, user::*}, - *, - }; + use crate::{comment::*, community::*, post::*, tests::establish_unpooled_connection, user::*}; + #[test] fn test_crud() { let conn = establish_unpooled_connection(); diff --git a/server/src/db/comment_view.rs b/server/lemmy_db/src/comment_view.rs similarity index 92% rename from server/src/db/comment_view.rs rename to server/lemmy_db/src/comment_view.rs index a37cdbcd..914e568c 100644 --- a/server/src/db/comment_view.rs +++ b/server/lemmy_db/src/comment_view.rs @@ -1,4 +1,5 @@ -use crate::db::{fuzzy_search, limit_and_offset, ListingType, MaybeOptional, SortType}; +// TODO, remove the cross join here, just join to user directly +use crate::{fuzzy_search, limit_and_offset, ListingType, MaybeOptional, SortType}; use diesel::{dsl::*, pg::Pg, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -26,6 +27,7 @@ table! { creator_actor_id -> Text, creator_local -> Bool, creator_name -> Varchar, + creator_published -> Timestamp, creator_avatar -> Nullable, score -> BigInt, upvotes -> BigInt, @@ -39,7 +41,7 @@ table! { } table! { - comment_mview (id) { + comment_fast_view (id) { id -> Int4, creator_id -> Int4, post_id -> Int4, @@ -61,6 +63,7 @@ table! { creator_actor_id -> Text, creator_local -> Bool, creator_name -> Varchar, + creator_published -> Timestamp, creator_avatar -> Nullable, score -> BigInt, upvotes -> BigInt, @@ -76,7 +79,7 @@ table! { #[derive( Queryable, Identifiable, PartialEq, Debug, Serialize, Deserialize, QueryableByName, Clone, )] -#[table_name = "comment_view"] +#[table_name = "comment_fast_view"] pub struct CommentView { pub id: i32, pub creator_id: i32, @@ -99,6 +102,7 @@ pub struct CommentView { pub creator_actor_id: String, pub creator_local: bool, pub creator_name: String, + pub creator_published: chrono::NaiveDateTime, pub creator_avatar: Option, pub score: i64, pub upvotes: i64, @@ -112,7 +116,7 @@ pub struct CommentView { pub struct CommentQueryBuilder<'a> { conn: &'a PgConnection, - query: super::comment_view::comment_mview::BoxedQuery<'a, Pg>, + query: super::comment_view::comment_fast_view::BoxedQuery<'a, Pg>, listing_type: ListingType, sort: &'a SortType, for_community_id: Option, @@ -127,9 +131,9 @@ pub struct CommentQueryBuilder<'a> { impl<'a> CommentQueryBuilder<'a> { pub fn create(conn: &'a PgConnection) -> Self { - use super::comment_view::comment_mview::dsl::*; + use super::comment_view::comment_fast_view::dsl::*; - let query = comment_mview.into_boxed(); + let query = comment_fast_view.into_boxed(); CommentQueryBuilder { conn, @@ -198,7 +202,7 @@ impl<'a> CommentQueryBuilder<'a> { } pub fn list(self) -> Result, Error> { - use super::comment_view::comment_mview::dsl::*; + use super::comment_view::comment_fast_view::dsl::*; let mut query = self.query; @@ -270,8 +274,8 @@ impl CommentView { from_comment_id: i32, my_user_id: Option, ) -> Result { - use super::comment_view::comment_mview::dsl::*; - let mut query = comment_mview.into_boxed(); + use super::comment_view::comment_fast_view::dsl::*; + let mut query = comment_fast_view.into_boxed(); // The view lets you pass a null user_id, if you're not logged in if let Some(my_user_id) = my_user_id { @@ -290,7 +294,7 @@ impl CommentView { // The faked schema since diesel doesn't do views table! { - reply_view (id) { + reply_fast_view (id) { id -> Int4, creator_id -> Int4, post_id -> Int4, @@ -313,6 +317,7 @@ table! { creator_local -> Bool, creator_name -> Varchar, creator_avatar -> Nullable, + creator_published -> Timestamp, score -> BigInt, upvotes -> BigInt, downvotes -> BigInt, @@ -328,7 +333,7 @@ table! { #[derive( Queryable, Identifiable, PartialEq, Debug, Serialize, Deserialize, QueryableByName, Clone, )] -#[table_name = "reply_view"] +#[table_name = "reply_fast_view"] pub struct ReplyView { pub id: i32, pub creator_id: i32, @@ -352,6 +357,7 @@ pub struct ReplyView { pub creator_local: bool, pub creator_name: String, pub creator_avatar: Option, + pub creator_published: chrono::NaiveDateTime, pub score: i64, pub upvotes: i64, pub downvotes: i64, @@ -365,7 +371,7 @@ pub struct ReplyView { pub struct ReplyQueryBuilder<'a> { conn: &'a PgConnection, - query: super::comment_view::reply_view::BoxedQuery<'a, Pg>, + query: super::comment_view::reply_fast_view::BoxedQuery<'a, Pg>, for_user_id: i32, sort: &'a SortType, unread_only: bool, @@ -375,9 +381,9 @@ pub struct ReplyQueryBuilder<'a> { impl<'a> ReplyQueryBuilder<'a> { pub fn create(conn: &'a PgConnection, for_user_id: i32) -> Self { - use super::comment_view::reply_view::dsl::*; + use super::comment_view::reply_fast_view::dsl::*; - let query = reply_view.into_boxed(); + let query = reply_fast_view.into_boxed(); ReplyQueryBuilder { conn, @@ -411,7 +417,7 @@ impl<'a> ReplyQueryBuilder<'a> { } pub fn list(self) -> Result, Error> { - use super::comment_view::reply_view::dsl::*; + use super::comment_view::reply_fast_view::dsl::*; let mut query = self.query; @@ -454,11 +460,17 @@ impl<'a> ReplyQueryBuilder<'a> { #[cfg(test)] mod tests { - use super::{ - super::{comment::*, community::*, post::*, user::*}, + use crate::{ + comment::*, + comment_view::*, + community::*, + post::*, + tests::establish_unpooled_connection, + user::*, + Crud, + Likeable, *, }; - use crate::db::{establish_unpooled_connection, Crud, Likeable}; #[test] fn test_crud() { @@ -575,6 +587,7 @@ mod tests { published: inserted_comment.published, updated: None, creator_name: inserted_user.name.to_owned(), + creator_published: inserted_user.published, creator_avatar: None, score: 1, downvotes: 0, @@ -608,6 +621,7 @@ mod tests { published: inserted_comment.published, updated: None, creator_name: inserted_user.name.to_owned(), + creator_published: inserted_user.published, creator_avatar: None, score: 1, downvotes: 0, @@ -615,8 +629,8 @@ mod tests { upvotes: 1, user_id: Some(inserted_user.id), my_vote: Some(1), - subscribed: None, - saved: None, + subscribed: Some(false), + saved: Some(false), ap_id: "http://fake.com".to_string(), local: true, community_actor_id: inserted_community.actor_id.to_owned(), diff --git a/server/src/db/community.rs b/server/lemmy_db/src/community.rs similarity index 98% rename from server/src/db/community.rs rename to server/lemmy_db/src/community.rs index 461ba473..60752080 100644 --- a/server/src/db/community.rs +++ b/server/lemmy_db/src/community.rs @@ -1,6 +1,9 @@ use crate::{ - db::{Bannable, Crud, Followable, Joinable}, schema::{community, community_follower, community_moderator, community_user_ban}, + Bannable, + Crud, + Followable, + Joinable, }; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -232,8 +235,7 @@ impl Followable for CommunityFollower { #[cfg(test)] mod tests { - use super::{super::user::*, *}; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; + use crate::{community::*, tests::establish_unpooled_connection, user::*, ListingType, SortType}; #[test] fn test_crud() { diff --git a/server/src/db/community_view.rs b/server/lemmy_db/src/community_view.rs similarity index 95% rename from server/src/db/community_view.rs rename to server/lemmy_db/src/community_view.rs index ea7b2a7c..b465ddab 100644 --- a/server/src/db/community_view.rs +++ b/server/lemmy_db/src/community_view.rs @@ -1,5 +1,5 @@ -use super::community_view::community_mview::BoxedQuery; -use crate::db::{fuzzy_search, limit_and_offset, MaybeOptional, SortType}; +use super::community_view::community_fast_view::BoxedQuery; +use crate::{fuzzy_search, limit_and_offset, MaybeOptional, SortType}; use diesel::{pg::Pg, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -34,7 +34,7 @@ table! { } table! { - community_mview (id) { + community_fast_view (id) { id -> Int4, name -> Varchar, title -> Varchar, @@ -114,7 +114,7 @@ table! { #[derive( Queryable, Identifiable, PartialEq, Debug, Serialize, Deserialize, QueryableByName, Clone, )] -#[table_name = "community_view"] +#[table_name = "community_fast_view"] pub struct CommunityView { pub id: i32, pub name: String, @@ -156,9 +156,9 @@ pub struct CommunityQueryBuilder<'a> { impl<'a> CommunityQueryBuilder<'a> { pub fn create(conn: &'a PgConnection) -> Self { - use super::community_view::community_mview::dsl::*; + use super::community_view::community_fast_view::dsl::*; - let query = community_mview.into_boxed(); + let query = community_fast_view.into_boxed(); CommunityQueryBuilder { conn, @@ -203,7 +203,7 @@ impl<'a> CommunityQueryBuilder<'a> { } pub fn list(self) -> Result, Error> { - use super::community_view::community_mview::dsl::*; + use super::community_view::community_fast_view::dsl::*; let mut query = self.query; @@ -259,9 +259,9 @@ impl CommunityView { from_community_id: i32, from_user_id: Option, ) -> Result { - use super::community_view::community_mview::dsl::*; + use super::community_view::community_fast_view::dsl::*; - let mut query = community_mview.into_boxed(); + let mut query = community_fast_view.into_boxed(); query = query.filter(id.eq(from_community_id)); diff --git a/server/src/db/mod.rs b/server/lemmy_db/src/lib.rs similarity index 78% rename from server/src/db/mod.rs rename to server/lemmy_db/src/lib.rs index da69f8dc..b34919cd 100644 --- a/server/src/db/mod.rs +++ b/server/lemmy_db/src/lib.rs @@ -1,10 +1,22 @@ -use crate::settings::Settings; +#[macro_use] +pub extern crate diesel; +#[macro_use] +pub extern crate strum_macros; +pub extern crate bcrypt; +pub extern crate chrono; +pub extern crate log; +pub extern crate serde; +pub extern crate serde_json; +pub extern crate sha2; +pub extern crate strum; + +use chrono::NaiveDateTime; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; +use std::{env, env::VarError}; pub mod activity; pub mod category; -pub mod code_migrations; pub mod comment; pub mod comment_view; pub mod community; @@ -16,6 +28,7 @@ pub mod post; pub mod post_view; pub mod private_message; pub mod private_message_view; +pub mod schema; pub mod site; pub mod site_view; pub mod user; @@ -111,9 +124,8 @@ impl MaybeOptional for Option { } } -pub fn establish_unpooled_connection() -> PgConnection { - let db_url = Settings::get().get_database_url(); - PgConnection::establish(&db_url).unwrap_or_else(|_| panic!("Error connecting to {}", db_url)) +pub fn get_database_url_from_env() -> Result { + env::var("LEMMY_DATABASE_URL") } #[derive(EnumString, ToString, Debug, Serialize, Deserialize)] @@ -155,9 +167,25 @@ pub fn limit_and_offset(page: Option, limit: Option) -> (i64, i64) { let offset = limit * (page - 1); (limit, offset) } + +pub fn naive_now() -> NaiveDateTime { + chrono::prelude::Utc::now().naive_utc() +} + #[cfg(test)] mod tests { use super::fuzzy_search; + use crate::get_database_url_from_env; + use diesel::{Connection, PgConnection}; + + pub fn establish_unpooled_connection() -> PgConnection { + let db_url = match get_database_url_from_env() { + Ok(url) => url, + Err(_) => panic!("Failed to read database URL from env var LEMMY_DATABASE_URL"), + }; + PgConnection::establish(&db_url).unwrap_or_else(|_| panic!("Error connecting to {}", db_url)) + } + #[test] fn test_fuzzy_search() { let test = "This is a fuzzy search"; diff --git a/server/src/db/moderator.rs b/server/lemmy_db/src/moderator.rs similarity index 99% rename from server/src/db/moderator.rs rename to server/lemmy_db/src/moderator.rs index 44b04ec6..f5d33d96 100644 --- a/server/src/db/moderator.rs +++ b/server/lemmy_db/src/moderator.rs @@ -1,5 +1,4 @@ use crate::{ - db::Crud, schema::{ mod_add, mod_add_community, @@ -11,6 +10,7 @@ use crate::{ mod_remove_post, mod_sticky_post, }, + Crud, }; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -437,11 +437,16 @@ impl Crud for ModAdd { #[cfg(test)] mod tests { - use super::{ - super::{comment::*, community::*, post::*, user::*}, - *, + use crate::{ + comment::*, + community::*, + moderator::*, + post::*, + tests::establish_unpooled_connection, + user::*, + ListingType, + SortType, }; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; // use Crud; #[test] diff --git a/server/src/db/moderator_views.rs b/server/lemmy_db/src/moderator_views.rs similarity index 99% rename from server/src/db/moderator_views.rs rename to server/lemmy_db/src/moderator_views.rs index f5b109fe..024907c3 100644 --- a/server/src/db/moderator_views.rs +++ b/server/lemmy_db/src/moderator_views.rs @@ -1,4 +1,4 @@ -use crate::db::limit_and_offset; +use crate::limit_and_offset; use diesel::{result::Error, *}; use serde::{Deserialize, Serialize}; diff --git a/server/src/db/password_reset_request.rs b/server/lemmy_db/src/password_reset_request.rs similarity index 98% rename from server/src/db/password_reset_request.rs rename to server/lemmy_db/src/password_reset_request.rs index 4a071f07..a2692add 100644 --- a/server/src/db/password_reset_request.rs +++ b/server/lemmy_db/src/password_reset_request.rs @@ -1,6 +1,6 @@ use crate::{ - db::Crud, schema::{password_reset_request, password_reset_request::dsl::*}, + Crud, }; use diesel::{dsl::*, result::Error, *}; use sha2::{Digest, Sha256}; @@ -82,7 +82,7 @@ impl PasswordResetRequest { #[cfg(test)] mod tests { use super::{super::user::*, *}; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; + use crate::{tests::establish_unpooled_connection, ListingType, SortType}; #[test] fn test_crud() { diff --git a/server/src/db/post.rs b/server/lemmy_db/src/post.rs similarity index 96% rename from server/src/db/post.rs rename to server/lemmy_db/src/post.rs index 91c1dcbf..1525a675 100644 --- a/server/src/db/post.rs +++ b/server/lemmy_db/src/post.rs @@ -1,8 +1,10 @@ use crate::{ - apub::{make_apub_endpoint, EndpointType}, - db::{Crud, Likeable, Readable, Saveable}, naive_now, schema::{post, post_like, post_read, post_saved}, + Crud, + Likeable, + Readable, + Saveable, }; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -75,12 +77,11 @@ impl Post { post.filter(ap_id.eq(object_id)).first::(conn) } - pub fn update_ap_id(conn: &PgConnection, post_id: i32) -> Result { + pub fn update_ap_id(conn: &PgConnection, post_id: i32, apub_id: String) -> Result { use crate::schema::post::dsl::*; - let apid = make_apub_endpoint(EndpointType::Post, &post_id.to_string()).to_string(); diesel::update(post.find(post_id)) - .set(ap_id.eq(apid)) + .set(ap_id.eq(apub_id)) .get_result::(conn) } @@ -241,11 +242,14 @@ impl Readable for PostRead { #[cfg(test)] mod tests { - use super::{ - super::{community::*, user::*}, - *, + use crate::{ + community::*, + post::*, + tests::establish_unpooled_connection, + user::*, + ListingType, + SortType, }; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; #[test] fn test_crud() { diff --git a/server/src/db/post_view.rs b/server/lemmy_db/src/post_view.rs similarity index 94% rename from server/src/db/post_view.rs rename to server/lemmy_db/src/post_view.rs index fbbf658d..b55359ea 100644 --- a/server/src/db/post_view.rs +++ b/server/lemmy_db/src/post_view.rs @@ -1,5 +1,5 @@ -use super::post_view::post_mview::BoxedQuery; -use crate::db::{fuzzy_search, limit_and_offset, ListingType, MaybeOptional, SortType}; +use super::post_view::post_fast_view::BoxedQuery; +use crate::{fuzzy_search, limit_and_offset, ListingType, MaybeOptional, SortType}; use diesel::{dsl::*, pg::Pg, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -25,12 +25,13 @@ table! { thumbnail_url -> Nullable, ap_id -> Text, local -> Bool, - banned -> Bool, - banned_from_community -> Bool, creator_actor_id -> Text, creator_local -> Bool, creator_name -> Varchar, + creator_published -> Timestamp, creator_avatar -> Nullable, + banned -> Bool, + banned_from_community -> Bool, community_actor_id -> Text, community_local -> Bool, community_name -> Varchar, @@ -52,7 +53,7 @@ table! { } table! { - post_mview (id) { + post_fast_view (id) { id -> Int4, name -> Varchar, url -> Nullable, @@ -72,12 +73,13 @@ table! { thumbnail_url -> Nullable, ap_id -> Text, local -> Bool, - banned -> Bool, - banned_from_community -> Bool, creator_actor_id -> Text, creator_local -> Bool, creator_name -> Varchar, + creator_published -> Timestamp, creator_avatar -> Nullable, + banned -> Bool, + banned_from_community -> Bool, community_actor_id -> Text, community_local -> Bool, community_name -> Varchar, @@ -101,7 +103,7 @@ table! { #[derive( Queryable, Identifiable, PartialEq, Debug, Serialize, Deserialize, QueryableByName, Clone, )] -#[table_name = "post_view"] +#[table_name = "post_fast_view"] pub struct PostView { pub id: i32, pub name: String, @@ -122,12 +124,13 @@ pub struct PostView { pub thumbnail_url: Option, pub ap_id: String, pub local: bool, - pub banned: bool, - pub banned_from_community: bool, pub creator_actor_id: String, pub creator_local: bool, pub creator_name: String, + pub creator_published: chrono::NaiveDateTime, pub creator_avatar: Option, + pub banned: bool, + pub banned_from_community: bool, pub community_actor_id: String, pub community_local: bool, pub community_name: String, @@ -166,9 +169,9 @@ pub struct PostQueryBuilder<'a> { impl<'a> PostQueryBuilder<'a> { pub fn create(conn: &'a PgConnection) -> Self { - use super::post_view::post_mview::dsl::*; + use super::post_view::post_fast_view::dsl::*; - let query = post_mview.into_boxed(); + let query = post_fast_view.into_boxed(); PostQueryBuilder { conn, @@ -249,7 +252,7 @@ impl<'a> PostQueryBuilder<'a> { } pub fn list(self) -> Result, Error> { - use super::post_view::post_mview::dsl::*; + use super::post_view::post_fast_view::dsl::*; let mut query = self.query; @@ -345,10 +348,10 @@ impl PostView { from_post_id: i32, my_user_id: Option, ) -> Result { - use super::post_view::post_mview::dsl::*; + use super::post_view::post_fast_view::dsl::*; use diesel::prelude::*; - let mut query = post_mview.into_boxed(); + let mut query = post_fast_view.into_boxed(); query = query.filter(id.eq(from_post_id)); @@ -364,11 +367,16 @@ impl PostView { #[cfg(test)] mod tests { - use super::{ - super::{community::*, post::*, user::*}, + use crate::{ + community::*, + post::*, + post_view::*, + tests::establish_unpooled_connection, + user::*, + Crud, + Likeable, *, }; - use crate::db::{establish_unpooled_connection, Crud, Likeable}; #[test] fn test_crud() { @@ -470,6 +478,25 @@ mod tests { score: 1, }; + let read_post_listings_with_user = PostQueryBuilder::create(&conn) + .listing_type(ListingType::Community) + .sort(&SortType::New) + .for_community_id(inserted_community.id) + .my_user_id(inserted_user.id) + .list() + .unwrap(); + + let read_post_listings_no_user = PostQueryBuilder::create(&conn) + .listing_type(ListingType::Community) + .sort(&SortType::New) + .for_community_id(inserted_community.id) + .list() + .unwrap(); + + let read_post_listing_no_user = PostView::read(&conn, inserted_post.id, None).unwrap(); + let read_post_listing_with_user = + PostView::read(&conn, inserted_post.id, Some(inserted_user.id)).unwrap(); + // the non user version let expected_post_listing_no_user = PostView { user_id: None, @@ -480,6 +507,7 @@ mod tests { body: None, creator_id: inserted_user.id, creator_name: user_name.to_owned(), + creator_published: inserted_user.published, creator_avatar: None, banned: false, banned_from_community: false, @@ -496,7 +524,7 @@ mod tests { score: 1, upvotes: 1, downvotes: 0, - hot_rank: 1728, + hot_rank: read_post_listing_no_user.hot_rank, published: inserted_post.published, newest_activity_time: inserted_post.published, updated: None, @@ -529,6 +557,7 @@ mod tests { stickied: false, creator_id: inserted_user.id, creator_name: user_name, + creator_published: inserted_user.published, creator_avatar: None, banned: false, banned_from_community: false, @@ -541,13 +570,13 @@ mod tests { score: 1, upvotes: 1, downvotes: 0, - hot_rank: 1728, + hot_rank: read_post_listing_with_user.hot_rank, published: inserted_post.published, newest_activity_time: inserted_post.published, updated: None, - subscribed: None, - read: None, - saved: None, + subscribed: Some(false), + read: Some(false), + saved: Some(false), nsfw: false, embed_title: None, embed_description: None, @@ -561,25 +590,6 @@ mod tests { community_local: true, }; - let read_post_listings_with_user = PostQueryBuilder::create(&conn) - .listing_type(ListingType::Community) - .sort(&SortType::New) - .for_community_id(inserted_community.id) - .my_user_id(inserted_user.id) - .list() - .unwrap(); - - let read_post_listings_no_user = PostQueryBuilder::create(&conn) - .listing_type(ListingType::Community) - .sort(&SortType::New) - .for_community_id(inserted_community.id) - .list() - .unwrap(); - - let read_post_listing_no_user = PostView::read(&conn, inserted_post.id, None).unwrap(); - let read_post_listing_with_user = - PostView::read(&conn, inserted_post.id, Some(inserted_user.id)).unwrap(); - let like_removed = PostLike::remove(&conn, &post_like_form).unwrap(); let num_deleted = Post::delete(&conn, inserted_post.id).unwrap(); Community::delete(&conn, inserted_community.id).unwrap(); diff --git a/server/src/db/private_message.rs b/server/lemmy_db/src/private_message.rs similarity index 92% rename from server/src/db/private_message.rs rename to server/lemmy_db/src/private_message.rs index 9d362bbf..1c0b455f 100644 --- a/server/src/db/private_message.rs +++ b/server/lemmy_db/src/private_message.rs @@ -1,8 +1,4 @@ -use crate::{ - apub::{make_apub_endpoint, EndpointType}, - db::Crud, - schema::private_message, -}; +use crate::{schema::private_message, Crud}; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -66,16 +62,15 @@ impl Crud for PrivateMessage { } impl PrivateMessage { - pub fn update_ap_id(conn: &PgConnection, private_message_id: i32) -> Result { + pub fn update_ap_id( + conn: &PgConnection, + private_message_id: i32, + apub_id: String, + ) -> Result { use crate::schema::private_message::dsl::*; - let apid = make_apub_endpoint( - EndpointType::PrivateMessage, - &private_message_id.to_string(), - ) - .to_string(); diesel::update(private_message.find(private_message_id)) - .set(ap_id.eq(apid)) + .set(ap_id.eq(apub_id)) .get_result::(conn) } @@ -89,8 +84,13 @@ impl PrivateMessage { #[cfg(test)] mod tests { - use super::{super::user::*, *}; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; + use crate::{ + private_message::*, + tests::establish_unpooled_connection, + user::*, + ListingType, + SortType, + }; #[test] fn test_crud() { diff --git a/server/src/db/private_message_view.rs b/server/lemmy_db/src/private_message_view.rs similarity index 79% rename from server/src/db/private_message_view.rs rename to server/lemmy_db/src/private_message_view.rs index 9a1df439..dfb11c44 100644 --- a/server/src/db/private_message_view.rs +++ b/server/lemmy_db/src/private_message_view.rs @@ -1,4 +1,4 @@ -use crate::db::{limit_and_offset, MaybeOptional}; +use crate::{limit_and_offset, MaybeOptional}; use diesel::{pg::Pg, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -26,29 +26,6 @@ table! { } } -table! { - private_message_mview (id) { - id -> Int4, - creator_id -> Int4, - recipient_id -> Int4, - content -> Text, - deleted -> Bool, - read -> Bool, - published -> Timestamp, - updated -> Nullable, - ap_id -> Text, - local -> Bool, - creator_name -> Varchar, - creator_avatar -> Nullable, - creator_actor_id -> Text, - creator_local -> Bool, - recipient_name -> Varchar, - recipient_avatar -> Nullable, - recipient_actor_id -> Text, - recipient_local -> Bool, - } -} - #[derive( Queryable, Identifiable, PartialEq, Debug, Serialize, Deserialize, QueryableByName, Clone, )] @@ -76,7 +53,7 @@ pub struct PrivateMessageView { pub struct PrivateMessageQueryBuilder<'a> { conn: &'a PgConnection, - query: super::private_message_view::private_message_mview::BoxedQuery<'a, Pg>, + query: super::private_message_view::private_message_view::BoxedQuery<'a, Pg>, for_recipient_id: i32, unread_only: bool, page: Option, @@ -85,9 +62,9 @@ pub struct PrivateMessageQueryBuilder<'a> { impl<'a> PrivateMessageQueryBuilder<'a> { pub fn create(conn: &'a PgConnection, for_recipient_id: i32) -> Self { - use super::private_message_view::private_message_mview::dsl::*; + use super::private_message_view::private_message_view::dsl::*; - let query = private_message_mview.into_boxed(); + let query = private_message_view.into_boxed(); PrivateMessageQueryBuilder { conn, @@ -115,7 +92,7 @@ impl<'a> PrivateMessageQueryBuilder<'a> { } pub fn list(self) -> Result, Error> { - use super::private_message_view::private_message_mview::dsl::*; + use super::private_message_view::private_message_view::dsl::*; let mut query = self.query.filter(deleted.eq(false)); diff --git a/server/src/schema.rs b/server/lemmy_db/src/schema.rs similarity index 68% rename from server/src/schema.rs rename to server/lemmy_db/src/schema.rs index 8096d301..18a522df 100644 --- a/server/src/schema.rs +++ b/server/lemmy_db/src/schema.rs @@ -33,6 +33,38 @@ table! { } } +table! { + comment_aggregates_fast (id) { + id -> Int4, + creator_id -> Nullable, + post_id -> Nullable, + parent_id -> Nullable, + content -> Nullable, + removed -> Nullable, + read -> Nullable, + published -> Nullable, + updated -> Nullable, + deleted -> Nullable, + ap_id -> Nullable, + local -> Nullable, + community_id -> Nullable, + community_actor_id -> Nullable, + community_local -> Nullable, + community_name -> Nullable, + banned -> Nullable, + banned_from_community -> Nullable, + creator_actor_id -> Nullable, + creator_local -> Nullable, + creator_name -> Nullable, + creator_published -> Nullable, + creator_avatar -> Nullable, + score -> Nullable, + upvotes -> Nullable, + downvotes -> Nullable, + hot_rank -> Nullable, + } +} + table! { comment_like (id) { id -> Int4, @@ -74,6 +106,34 @@ table! { } } +table! { + community_aggregates_fast (id) { + id -> Int4, + name -> Nullable, + title -> Nullable, + description -> Nullable, + category_id -> Nullable, + creator_id -> Nullable, + removed -> Nullable, + published -> Nullable, + updated -> Nullable, + deleted -> Nullable, + nsfw -> Nullable, + actor_id -> Nullable, + local -> Nullable, + last_refreshed_at -> Nullable, + creator_actor_id -> Nullable, + creator_local -> Nullable, + creator_name -> Nullable, + creator_avatar -> Nullable, + category_name -> Nullable, + number_of_subscribers -> Nullable, + number_of_posts -> Nullable, + number_of_comments -> Nullable, + hot_rank -> Nullable, + } +} + table! { community_follower (id) { id -> Int4, @@ -234,6 +294,49 @@ table! { } } +table! { + post_aggregates_fast (id) { + id -> Int4, + name -> Nullable, + url -> Nullable, + body -> Nullable, + creator_id -> Nullable, + community_id -> Nullable, + removed -> Nullable, + locked -> Nullable, + published -> Nullable, + updated -> Nullable, + deleted -> Nullable, + nsfw -> Nullable, + stickied -> Nullable, + embed_title -> Nullable, + embed_description -> Nullable, + embed_html -> Nullable, + thumbnail_url -> Nullable, + ap_id -> Nullable, + local -> Nullable, + creator_actor_id -> Nullable, + creator_local -> Nullable, + creator_name -> Nullable, + creator_published -> Nullable, + creator_avatar -> Nullable, + banned -> Nullable, + banned_from_community -> Nullable, + community_actor_id -> Nullable, + community_local -> Nullable, + community_name -> Nullable, + community_removed -> Nullable, + community_deleted -> Nullable, + community_nsfw -> Nullable, + number_of_comments -> Nullable, + score -> Nullable, + upvotes -> Nullable, + downvotes -> Nullable, + hot_rank -> Nullable, + newest_activity_time -> Nullable, + } +} + table! { post_like (id) { id -> Int4, @@ -328,6 +431,28 @@ table! { } } +table! { + user_fast (id) { + id -> Int4, + actor_id -> Nullable, + name -> Nullable, + avatar -> Nullable, + email -> Nullable, + matrix_user_id -> Nullable, + bio -> Nullable, + local -> Nullable, + admin -> Nullable, + banned -> Nullable, + show_avatars -> Nullable, + send_notifications_to_email -> Nullable, + published -> Nullable, + number_of_posts -> Nullable, + post_score -> Nullable, + number_of_comments -> Nullable, + comment_score -> Nullable, + } +} + table! { user_mention (id) { id -> Int4, @@ -384,9 +509,11 @@ allow_tables_to_appear_in_same_query!( activity, category, comment, + comment_aggregates_fast, comment_like, comment_saved, community, + community_aggregates_fast, community_follower, community_moderator, community_user_ban, @@ -401,6 +528,7 @@ allow_tables_to_appear_in_same_query!( mod_sticky_post, password_reset_request, post, + post_aggregates_fast, post_like, post_read, post_saved, @@ -408,5 +536,6 @@ allow_tables_to_appear_in_same_query!( site, user_, user_ban, + user_fast, user_mention, ); diff --git a/server/src/db/site.rs b/server/lemmy_db/src/site.rs similarity index 97% rename from server/src/db/site.rs rename to server/lemmy_db/src/site.rs index c752bfe7..066ae0b1 100644 --- a/server/src/db/site.rs +++ b/server/lemmy_db/src/site.rs @@ -1,4 +1,4 @@ -use crate::{db::Crud, schema::site}; +use crate::{schema::site, Crud}; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; diff --git a/server/src/db/site_view.rs b/server/lemmy_db/src/site_view.rs similarity index 100% rename from server/src/db/site_view.rs rename to server/lemmy_db/src/site_view.rs diff --git a/server/src/db/user.rs b/server/lemmy_db/src/user.rs similarity index 73% rename from server/src/db/user.rs rename to server/lemmy_db/src/user.rs index 4ca0a041..556fc1a7 100644 --- a/server/src/db/user.rs +++ b/server/lemmy_db/src/user.rs @@ -1,14 +1,10 @@ use crate::{ - db::Crud, - is_email_regex, naive_now, schema::{user_, user_::dsl::*}, - settings::Settings, + Crud, }; use bcrypt::{hash, DEFAULT_COST}; use diesel::{dsl::*, result::Error, *}; -use jsonwebtoken::{decode, encode, DecodingKey, EncodingKey, Header, TokenData, Validation}; -use serde::{Deserialize, Serialize}; #[derive(Clone, Queryable, Identifiable, PartialEq, Debug)] #[table_name = "user_"] @@ -131,90 +127,23 @@ impl User_ { } } -#[derive(Debug, Serialize, Deserialize)] -pub struct Claims { - pub id: i32, - pub username: String, - pub iss: String, - pub show_nsfw: bool, - pub theme: String, - pub default_sort_type: i16, - pub default_listing_type: i16, - pub lang: String, - pub avatar: Option, - pub show_avatars: bool, -} - -impl Claims { - pub fn decode(jwt: &str) -> Result, jsonwebtoken::errors::Error> { - let v = Validation { - validate_exp: false, - ..Validation::default() - }; - decode::( - &jwt, - &DecodingKey::from_secret(Settings::get().jwt_secret.as_ref()), - &v, - ) - } -} - -type Jwt = String; impl User_ { - pub fn jwt(&self) -> Jwt { - let my_claims = Claims { - id: self.id, - username: self.name.to_owned(), - iss: Settings::get().hostname, - show_nsfw: self.show_nsfw, - theme: self.theme.to_owned(), - default_sort_type: self.default_sort_type, - default_listing_type: self.default_listing_type, - lang: self.lang.to_owned(), - avatar: self.avatar.to_owned(), - show_avatars: self.show_avatars.to_owned(), - }; - encode( - &Header::default(), - &my_claims, - &EncodingKey::from_secret(Settings::get().jwt_secret.as_ref()), - ) - .unwrap() - } - - pub fn find_by_username(conn: &PgConnection, username: &str) -> Result { + pub fn find_by_username(conn: &PgConnection, username: &str) -> Result { user_.filter(name.eq(username)).first::(conn) } - pub fn find_by_email(conn: &PgConnection, from_email: &str) -> Result { + pub fn find_by_email(conn: &PgConnection, from_email: &str) -> Result { user_.filter(email.eq(from_email)).first::(conn) } - pub fn find_by_email_or_username( - conn: &PgConnection, - username_or_email: &str, - ) -> Result { - if is_email_regex(username_or_email) { - User_::find_by_email(conn, username_or_email) - } else { - User_::find_by_username(conn, username_or_email) - } - } - - pub fn get_profile_url(&self) -> String { - format!("https://{}/u/{}", Settings::get().hostname, self.name) - } - - pub fn find_by_jwt(conn: &PgConnection, jwt: &str) -> Result { - let claims: Claims = Claims::decode(&jwt).expect("Invalid token").claims; - Self::read(&conn, claims.id) + pub fn get_profile_url(&self, hostname: &str) -> String { + format!("https://{}/u/{}", hostname, self.name) } } #[cfg(test)] mod tests { - use super::{User_, *}; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; + use crate::{tests::establish_unpooled_connection, user::*, ListingType, SortType}; #[test] fn test_crud() { diff --git a/server/src/db/user_mention.rs b/server/lemmy_db/src/user_mention.rs similarity index 96% rename from server/src/db/user_mention.rs rename to server/lemmy_db/src/user_mention.rs index 1d54fa98..9f23f441 100644 --- a/server/src/db/user_mention.rs +++ b/server/lemmy_db/src/user_mention.rs @@ -1,5 +1,5 @@ use super::comment::Comment; -use crate::{db::Crud, schema::user_mention}; +use crate::{schema::user_mention, Crud}; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -54,11 +54,16 @@ impl Crud for UserMention { #[cfg(test)] mod tests { - use super::{ - super::{comment::*, community::*, post::*, user::*}, - *, + use crate::{ + comment::*, + community::*, + post::*, + tests::establish_unpooled_connection, + user::*, + user_mention::*, + ListingType, + SortType, }; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; #[test] fn test_crud() { diff --git a/server/src/db/user_mention_view.rs b/server/lemmy_db/src/user_mention_view.rs similarity index 91% rename from server/src/db/user_mention_view.rs rename to server/lemmy_db/src/user_mention_view.rs index 100445b9..8bfbf453 100644 --- a/server/src/db/user_mention_view.rs +++ b/server/lemmy_db/src/user_mention_view.rs @@ -1,4 +1,4 @@ -use crate::db::{limit_and_offset, MaybeOptional, SortType}; +use crate::{limit_and_offset, MaybeOptional, SortType}; use diesel::{dsl::*, pg::Pg, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -40,7 +40,7 @@ table! { } table! { - user_mention_mview (id) { + user_mention_fast_view (id) { id -> Int4, user_mention_id -> Int4, creator_id -> Int4, @@ -78,7 +78,7 @@ table! { #[derive( Queryable, Identifiable, PartialEq, Debug, Serialize, Deserialize, QueryableByName, Clone, )] -#[table_name = "user_mention_view"] +#[table_name = "user_mention_fast_view"] pub struct UserMentionView { pub id: i32, pub user_mention_id: i32, @@ -115,7 +115,7 @@ pub struct UserMentionView { pub struct UserMentionQueryBuilder<'a> { conn: &'a PgConnection, - query: super::user_mention_view::user_mention_mview::BoxedQuery<'a, Pg>, + query: super::user_mention_view::user_mention_fast_view::BoxedQuery<'a, Pg>, for_user_id: i32, sort: &'a SortType, unread_only: bool, @@ -125,9 +125,9 @@ pub struct UserMentionQueryBuilder<'a> { impl<'a> UserMentionQueryBuilder<'a> { pub fn create(conn: &'a PgConnection, for_user_id: i32) -> Self { - use super::user_mention_view::user_mention_mview::dsl::*; + use super::user_mention_view::user_mention_fast_view::dsl::*; - let query = user_mention_mview.into_boxed(); + let query = user_mention_fast_view.into_boxed(); UserMentionQueryBuilder { conn, @@ -161,7 +161,7 @@ impl<'a> UserMentionQueryBuilder<'a> { } pub fn list(self) -> Result, Error> { - use super::user_mention_view::user_mention_mview::dsl::*; + use super::user_mention_view::user_mention_fast_view::dsl::*; let mut query = self.query; @@ -208,9 +208,9 @@ impl UserMentionView { from_user_mention_id: i32, from_recipient_id: i32, ) -> Result { - use super::user_mention_view::user_mention_view::dsl::*; + use super::user_mention_view::user_mention_fast_view::dsl::*; - user_mention_view + user_mention_fast_view .filter(user_mention_id.eq(from_user_mention_id)) .filter(user_id.eq(from_recipient_id)) .first::(conn) diff --git a/server/src/db/user_view.rs b/server/lemmy_db/src/user_view.rs similarity index 85% rename from server/src/db/user_view.rs rename to server/lemmy_db/src/user_view.rs index 57e2a4c9..84feba38 100644 --- a/server/src/db/user_view.rs +++ b/server/lemmy_db/src/user_view.rs @@ -1,5 +1,5 @@ -use super::user_view::user_mview::BoxedQuery; -use crate::db::{fuzzy_search, limit_and_offset, MaybeOptional, SortType}; +use super::user_view::user_fast::BoxedQuery; +use crate::{fuzzy_search, limit_and_offset, MaybeOptional, SortType}; use diesel::{dsl::*, pg::Pg, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -26,7 +26,7 @@ table! { } table! { - user_mview (id) { + user_fast (id) { id -> Int4, actor_id -> Text, name -> Varchar, @@ -50,7 +50,7 @@ table! { #[derive( Queryable, Identifiable, PartialEq, Debug, Serialize, Deserialize, QueryableByName, Clone, )] -#[table_name = "user_view"] +#[table_name = "user_fast"] pub struct UserView { pub id: i32, pub actor_id: String, @@ -81,9 +81,9 @@ pub struct UserQueryBuilder<'a> { impl<'a> UserQueryBuilder<'a> { pub fn create(conn: &'a PgConnection) -> Self { - use super::user_view::user_mview::dsl::*; + use super::user_view::user_fast::dsl::*; - let query = user_mview.into_boxed(); + let query = user_fast.into_boxed(); UserQueryBuilder { conn, @@ -100,7 +100,7 @@ impl<'a> UserQueryBuilder<'a> { } pub fn search_term>(mut self, search_term: T) -> Self { - use super::user_view::user_mview::dsl::*; + use super::user_view::user_fast::dsl::*; if let Some(search_term) = search_term.get_optional() { self.query = self.query.filter(name.ilike(fuzzy_search(&search_term))); } @@ -118,7 +118,7 @@ impl<'a> UserQueryBuilder<'a> { } pub fn list(self) -> Result, Error> { - use super::user_view::user_mview::dsl::*; + use super::user_view::user_fast::dsl::*; let mut query = self.query; @@ -151,17 +151,17 @@ impl<'a> UserQueryBuilder<'a> { impl UserView { pub fn read(conn: &PgConnection, from_user_id: i32) -> Result { - use super::user_view::user_mview::dsl::*; - user_mview.find(from_user_id).first::(conn) + use super::user_view::user_fast::dsl::*; + user_fast.find(from_user_id).first::(conn) } pub fn admins(conn: &PgConnection) -> Result, Error> { - use super::user_view::user_mview::dsl::*; - user_mview.filter(admin.eq(true)).load::(conn) + use super::user_view::user_fast::dsl::*; + user_fast.filter(admin.eq(true)).load::(conn) } pub fn banned(conn: &PgConnection) -> Result, Error> { - use super::user_view::user_mview::dsl::*; - user_mview.filter(banned.eq(true)).load::(conn) + use super::user_view::user_fast::dsl::*; + user_fast.filter(banned.eq(true)).load::(conn) } } diff --git a/server/lemmy_utils/Cargo.toml b/server/lemmy_utils/Cargo.toml new file mode 100644 index 00000000..fed22f58 --- /dev/null +++ b/server/lemmy_utils/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "lemmy_utils" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +regex = "1.3.5" +config = { version = "0.10.1", default-features = false, features = ["hjson"] } +chrono = { version = "0.4.7", features = ["serde"] } +lettre = "0.9.3" +lettre_email = "0.9.4" +log = "0.4.0" +itertools = "0.9.0" +rand = "0.7.3" +serde = { version = "1.0.105", features = ["derive"] } +serde_json = { version = "1.0.52", features = ["preserve_order"]} +comrak = "0.7" +lazy_static = "1.3.0" +openssl = "0.10" +url = { version = "2.1.1", features = ["serde"] } \ No newline at end of file diff --git a/server/lemmy_utils/src/lib.rs b/server/lemmy_utils/src/lib.rs new file mode 100644 index 00000000..bbee8500 --- /dev/null +++ b/server/lemmy_utils/src/lib.rs @@ -0,0 +1,339 @@ +#[macro_use] +pub extern crate lazy_static; +pub extern crate comrak; +pub extern crate lettre; +pub extern crate lettre_email; +pub extern crate openssl; +pub extern crate rand; +pub extern crate regex; +pub extern crate serde_json; +pub extern crate url; + +pub mod settings; + +use crate::settings::Settings; +use chrono::{DateTime, FixedOffset, Local, NaiveDateTime, Utc}; +use itertools::Itertools; +use lettre::{ + smtp::{ + authentication::{Credentials, Mechanism}, + extension::ClientId, + ConnectionReuseParameters, + }, + ClientSecurity, + SmtpClient, + Transport, +}; +use lettre_email::Email; +use openssl::{pkey::PKey, rsa::Rsa}; +use rand::{distributions::Alphanumeric, thread_rng, Rng}; +use regex::{Regex, RegexBuilder}; +use std::io::{Error, ErrorKind}; +use url::Url; + +pub fn to_datetime_utc(ndt: NaiveDateTime) -> DateTime { + DateTime::::from_utc(ndt, Utc) +} + +pub fn naive_from_unix(time: i64) -> NaiveDateTime { + NaiveDateTime::from_timestamp(time, 0) +} + +pub fn convert_datetime(datetime: NaiveDateTime) -> DateTime { + let now = Local::now(); + DateTime::::from_utc(datetime, *now.offset()) +} + +pub fn is_email_regex(test: &str) -> bool { + EMAIL_REGEX.is_match(test) +} + +pub fn remove_slurs(test: &str) -> String { + SLUR_REGEX.replace_all(test, "*removed*").to_string() +} + +pub fn slur_check(test: &str) -> Result<(), Vec<&str>> { + let mut matches: Vec<&str> = SLUR_REGEX.find_iter(test).map(|mat| mat.as_str()).collect(); + + // Unique + matches.sort_unstable(); + matches.dedup(); + + if matches.is_empty() { + Ok(()) + } else { + Err(matches) + } +} + +pub fn slurs_vec_to_str(slurs: Vec<&str>) -> String { + let start = "No slurs - "; + let combined = &slurs.join(", "); + [start, combined].concat() +} + +pub fn generate_random_string() -> String { + thread_rng().sample_iter(&Alphanumeric).take(30).collect() +} + +pub fn send_email( + subject: &str, + to_email: &str, + to_username: &str, + html: &str, +) -> Result<(), String> { + let email_config = Settings::get().email.ok_or("no_email_setup")?; + + let email = Email::builder() + .to((to_email, to_username)) + .from(email_config.smtp_from_address.to_owned()) + .subject(subject) + .html(html) + .build() + .unwrap(); + + let mailer = if email_config.use_tls { + SmtpClient::new_simple(&email_config.smtp_server).unwrap() + } else { + SmtpClient::new(&email_config.smtp_server, ClientSecurity::None).unwrap() + } + .hello_name(ClientId::Domain(Settings::get().hostname)) + .smtp_utf8(true) + .authentication_mechanism(Mechanism::Plain) + .connection_reuse(ConnectionReuseParameters::ReuseUnlimited); + let mailer = if let (Some(login), Some(password)) = + (&email_config.smtp_login, &email_config.smtp_password) + { + mailer.credentials(Credentials::new(login.to_owned(), password.to_owned())) + } else { + mailer + }; + + let mut transport = mailer.transport(); + let result = transport.send(email.into()); + transport.close(); + + match result { + Ok(_) => Ok(()), + Err(e) => Err(e.to_string()), + } +} + +pub fn markdown_to_html(text: &str) -> String { + comrak::markdown_to_html(text, &comrak::ComrakOptions::default()) +} + +// TODO nothing is done with community / group webfingers yet, so just ignore those for now +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct MentionData { + pub name: String, + pub domain: String, +} + +impl MentionData { + pub fn is_local(&self) -> bool { + Settings::get().hostname.eq(&self.domain) + } + pub fn full_name(&self) -> String { + format!("@{}@{}", &self.name, &self.domain) + } +} + +pub fn scrape_text_for_mentions(text: &str) -> Vec { + let mut out: Vec = Vec::new(); + for caps in MENTIONS_REGEX.captures_iter(text) { + out.push(MentionData { + name: caps["name"].to_string(), + domain: caps["domain"].to_string(), + }); + } + out.into_iter().unique().collect() +} + +pub fn is_valid_username(name: &str) -> bool { + VALID_USERNAME_REGEX.is_match(name) +} + +pub fn is_valid_community_name(name: &str) -> bool { + VALID_COMMUNITY_NAME_REGEX.is_match(name) +} + +pub fn is_valid_post_title(title: &str) -> bool { + VALID_POST_TITLE_REGEX.is_match(title) +} + +#[cfg(test)] +mod tests { + use crate::{ + is_email_regex, + is_valid_community_name, + is_valid_username, + is_valid_post_title, + remove_slurs, + scrape_text_for_mentions, + slur_check, + slurs_vec_to_str, + }; + + #[test] + fn test_mentions_regex() { + let text = "Just read a great blog post by [@tedu@honk.teduangst.com](/u/test). And another by !test_community@fish.teduangst.com . Another [@lemmy@lemmy-alpha:8540](/u/fish)"; + let mentions = scrape_text_for_mentions(text); + + assert_eq!(mentions[0].name, "tedu".to_string()); + assert_eq!(mentions[0].domain, "honk.teduangst.com".to_string()); + assert_eq!(mentions[1].domain, "lemmy-alpha:8540".to_string()); + } + + #[test] + fn test_email() { + assert!(is_email_regex("gush@gmail.com")); + assert!(!is_email_regex("nada_neutho")); + } + + #[test] + fn test_valid_register_username() { + assert!(is_valid_username("Hello_98")); + assert!(is_valid_username("ten")); + assert!(!is_valid_username("Hello-98")); + assert!(!is_valid_username("a")); + assert!(!is_valid_username("")); + } + + #[test] + fn test_valid_community_name() { + assert!(is_valid_community_name("example")); + assert!(is_valid_community_name("example_community")); + assert!(!is_valid_community_name("Example")); + assert!(!is_valid_community_name("Ex")); + assert!(!is_valid_community_name("")); + } + + #[test] + fn test_valid_post_title() { + assert!(is_valid_post_title("Post Title")); + assert!(is_valid_post_title(" POST TITLE 😃😃😃😃😃")); + assert!(!is_valid_post_title("\n \n \n \n ")); // tabs/spaces/newlines + } + + + + #[test] + fn test_slur_filter() { + let test = + "coons test dindu ladyboy tranny retardeds. Capitalized Niggerz. This is a bunch of other safe text."; + let slur_free = "No slurs here"; + assert_eq!( + remove_slurs(&test), + "*removed* test *removed* *removed* *removed* *removed*. Capitalized *removed*. This is a bunch of other safe text." + .to_string() + ); + + let has_slurs_vec = vec![ + "Niggerz", + "coons", + "dindu", + "ladyboy", + "retardeds", + "tranny", + ]; + let has_slurs_err_str = "No slurs - Niggerz, coons, dindu, ladyboy, retardeds, tranny"; + + assert_eq!(slur_check(test), Err(has_slurs_vec)); + assert_eq!(slur_check(slur_free), Ok(())); + if let Err(slur_vec) = slur_check(test) { + assert_eq!(&slurs_vec_to_str(slur_vec), has_slurs_err_str); + } + } + + // These helped with testing + // #[test] + // fn test_send_email() { + // let result = send_email("not a subject", "test_email@gmail.com", "ur user", "

HI there

"); + // assert!(result.is_ok()); + // } +} + +lazy_static! { + static ref EMAIL_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$").unwrap(); + static ref SLUR_REGEX: Regex = RegexBuilder::new(r"(fag(g|got|tard)?|maricos?|cock\s?sucker(s|ing)?|\bn(i|1)g(\b|g?(a|er)?(s|z)?)\b|dindu(s?)|mudslime?s?|kikes?|mongoloids?|towel\s*heads?|\bspi(c|k)s?\b|\bchinks?|niglets?|beaners?|\bnips?\b|\bcoons?\b|jungle\s*bunn(y|ies?)|jigg?aboo?s?|\bpakis?\b|rag\s*heads?|gooks?|cunts?|bitch(es|ing|y)?|puss(y|ies?)|twats?|feminazis?|whor(es?|ing)|\bslut(s|t?y)?|\btr(a|@)nn?(y|ies?)|ladyboy(s?)|\b(b|re|r)tard(ed)?s?)").case_insensitive(true).build().unwrap(); + static ref USERNAME_MATCHES_REGEX: Regex = Regex::new(r"/u/[a-zA-Z][0-9a-zA-Z_]*").unwrap(); + // TODO keep this old one, it didn't work with port well tho + // static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P[\w.]+)@(?P[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)").unwrap(); + static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P[\w.]+)@(?P[a-zA-Z0-9._:-]+)").unwrap(); + static ref VALID_USERNAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_]{3,20}$").unwrap(); + static ref VALID_COMMUNITY_NAME_REGEX: Regex = Regex::new(r"^[a-z0-9_]{3,20}$").unwrap(); + static ref VALID_POST_TITLE_REGEX: Regex = Regex::new(r".*\S.*").unwrap(); + pub static ref WEBFINGER_COMMUNITY_REGEX: Regex = Regex::new(&format!( + "^group:([a-z0-9_]{{3, 20}})@{}$", + Settings::get().hostname + )) + .unwrap(); + pub static ref WEBFINGER_USER_REGEX: Regex = Regex::new(&format!( + "^acct:([a-z0-9_]{{3, 20}})@{}$", + Settings::get().hostname + )) + .unwrap(); + pub static ref CACHE_CONTROL_REGEX: Regex = + Regex::new("^((text|image)/.+|application/javascript)$").unwrap(); +} + +pub struct Keypair { + pub private_key: String, + pub public_key: String, +} + +/// Generate the asymmetric keypair for ActivityPub HTTP signatures. +pub fn generate_actor_keypair() -> Result { + let rsa = Rsa::generate(2048)?; + let pkey = PKey::from_rsa(rsa)?; + let public_key = pkey.public_key_to_pem()?; + let private_key = pkey.private_key_to_pem_pkcs8()?; + let key_to_string = |key| match String::from_utf8(key) { + Ok(s) => Ok(s), + Err(e) => Err(Error::new( + ErrorKind::Other, + format!("Failed converting key to string: {}", e), + )), + }; + Ok(Keypair { + private_key: key_to_string(private_key)?, + public_key: key_to_string(public_key)?, + }) +} + +pub enum EndpointType { + Community, + User, + Post, + Comment, + PrivateMessage, +} + +pub fn get_apub_protocol_string() -> &'static str { + if Settings::get().federation.tls_enabled { + "https" + } else { + "http" + } +} + +/// Generates the ActivityPub ID for a given object type and ID. +pub fn make_apub_endpoint(endpoint_type: EndpointType, name: &str) -> Url { + let point = match endpoint_type { + EndpointType::Community => "c", + EndpointType::User => "u", + EndpointType::Post => "post", + EndpointType::Comment => "comment", + EndpointType::PrivateMessage => "private_message", + }; + + Url::parse(&format!( + "{}://{}/{}/{}", + get_apub_protocol_string(), + Settings::get().hostname, + point, + name + )) + .unwrap() +} diff --git a/server/src/settings.rs b/server/lemmy_utils/src/settings.rs similarity index 82% rename from server/src/settings.rs rename to server/lemmy_utils/src/settings.rs index 12ffacea..2ce33f58 100644 --- a/server/src/settings.rs +++ b/server/lemmy_utils/src/settings.rs @@ -1,7 +1,6 @@ -use crate::LemmyError; use config::{Config, ConfigError, Environment, File}; use serde::Deserialize; -use std::{env, fs, net::IpAddr, sync::RwLock}; +use std::{fs, io::Error, net::IpAddr, sync::RwLock}; static CONFIG_FILE_DEFAULTS: &str = "config/defaults.hjson"; static CONFIG_FILE: &str = "config/config.hjson"; @@ -76,6 +75,9 @@ impl Settings { /// First, defaults are loaded from CONFIG_FILE_DEFAULTS, then these values can be overwritten /// from CONFIG_FILE (optional). Finally, values from the environment (with prefix LEMMY) are /// added to the config. + /// + /// Note: The env var `LEMMY_DATABASE_URL` is parsed in + /// `server/lemmy_db/src/lib.rs::get_database_url_from_env()` fn init() -> Result { let mut s = Config::new(); @@ -98,31 +100,26 @@ impl Settings { SETTINGS.read().unwrap().to_owned() } - /// Returns the postgres connection url. If LEMMY_DATABASE_URL is set, that is used, - /// otherwise the connection url is generated from the config. pub fn get_database_url(&self) -> String { - match env::var("LEMMY_DATABASE_URL") { - Ok(url) => url, - Err(_) => format!( - "postgres://{}:{}@{}:{}/{}", - self.database.user, - self.database.password, - self.database.host, - self.database.port, - self.database.database - ), - } + format!( + "postgres://{}:{}@{}:{}/{}", + self.database.user, + self.database.password, + self.database.host, + self.database.port, + self.database.database + ) } pub fn api_endpoint(&self) -> String { format!("{}/api/v1", self.hostname) } - pub fn read_config_file() -> Result { - Ok(fs::read_to_string(CONFIG_FILE)?) + pub fn read_config_file() -> Result { + fs::read_to_string(CONFIG_FILE) } - pub fn save_config_file(data: &str) -> Result { + pub fn save_config_file(data: &str) -> Result { fs::write(CONFIG_FILE, data)?; // Reload the new settings diff --git a/server/migrations/2020-06-30-135809_remove_mat_views/down.sql b/server/migrations/2020-06-30-135809_remove_mat_views/down.sql new file mode 100644 index 00000000..5f72b76d --- /dev/null +++ b/server/migrations/2020-06-30-135809_remove_mat_views/down.sql @@ -0,0 +1,535 @@ +-- Dropping all the fast tables +drop table user_fast; +drop view post_fast_view; +drop table post_aggregates_fast; +drop view community_fast_view; +drop table community_aggregates_fast; +drop view reply_fast_view; +drop view user_mention_fast_view; +drop view comment_fast_view; +drop table comment_aggregates_fast; + +-- Re-adding all the triggers, functions, and mviews + +-- private message +create materialized view private_message_mview as select * from private_message_view; + +create unique index idx_private_message_mview_id on private_message_mview (id); + + +-- Create the triggers +create or replace function refresh_private_message() +returns trigger language plpgsql +as $$ +begin + refresh materialized view concurrently private_message_mview; + return null; +end $$; + +create trigger refresh_private_message +after insert or update or delete or truncate +on private_message +for each statement +execute procedure refresh_private_message(); + +-- user +create or replace function refresh_user() +returns trigger language plpgsql +as $$ +begin + refresh materialized view concurrently user_mview; + refresh materialized view concurrently comment_aggregates_mview; -- cause of bans + refresh materialized view concurrently post_aggregates_mview; + return null; +end $$; + +drop trigger refresh_user on user_; +create trigger refresh_user +after insert or update or delete or truncate +on user_ +for each statement +execute procedure refresh_user(); +drop view user_view cascade; + +create view user_view as +select +u.id, +u.actor_id, +u.name, +u.avatar, +u.email, +u.matrix_user_id, +u.bio, +u.local, +u.admin, +u.banned, +u.show_avatars, +u.send_notifications_to_email, +u.published, +(select count(*) from post p where p.creator_id = u.id) as number_of_posts, +(select coalesce(sum(score), 0) from post p, post_like pl where u.id = p.creator_id and p.id = pl.post_id) as post_score, +(select count(*) from comment c where c.creator_id = u.id) as number_of_comments, +(select coalesce(sum(score), 0) from comment c, comment_like cl where u.id = c.creator_id and c.id = cl.comment_id) as comment_score +from user_ u; + +create materialized view user_mview as select * from user_view; + +create unique index idx_user_mview_id on user_mview (id); + +-- community +drop trigger refresh_community on community; +create trigger refresh_community +after insert or update or delete or truncate +on community +for each statement +execute procedure refresh_community(); + +create or replace function refresh_community() +returns trigger language plpgsql +as $$ +begin + refresh materialized view concurrently post_aggregates_mview; + refresh materialized view concurrently community_aggregates_mview; + refresh materialized view concurrently user_mview; + return null; +end $$; + +drop view community_aggregates_view cascade; +create view community_aggregates_view as +-- Now that there's public and private keys, you have to be explicit here +select c.id, +c.name, +c.title, +c.description, +c.category_id, +c.creator_id, +c.removed, +c.published, +c.updated, +c.deleted, +c.nsfw, +c.actor_id, +c.local, +c.last_refreshed_at, +(select actor_id from user_ u where c.creator_id = u.id) as creator_actor_id, +(select local from user_ u where c.creator_id = u.id) as creator_local, +(select name from user_ u where c.creator_id = u.id) as creator_name, +(select avatar from user_ u where c.creator_id = u.id) as creator_avatar, +(select name from category ct where c.category_id = ct.id) as category_name, +(select count(*) from community_follower cf where cf.community_id = c.id) as number_of_subscribers, +(select count(*) from post p where p.community_id = c.id) as number_of_posts, +(select count(*) from comment co, post p where c.id = p.community_id and p.id = co.post_id) as number_of_comments, +hot_rank((select count(*) from community_follower cf where cf.community_id = c.id), c.published) as hot_rank +from community c; + +create materialized view community_aggregates_mview as select * from community_aggregates_view; + +create unique index idx_community_aggregates_mview_id on community_aggregates_mview (id); + +create view community_view as +with all_community as +( + select + ca.* + from community_aggregates_view ca +) + +select +ac.*, +u.id as user_id, +(select cf.id::boolean from community_follower cf where u.id = cf.user_id and ac.id = cf.community_id) as subscribed +from user_ u +cross join all_community ac + +union all + +select +ac.*, +null as user_id, +null as subscribed +from all_community ac +; + +create view community_mview as +with all_community as +( + select + ca.* + from community_aggregates_mview ca +) + +select +ac.*, +u.id as user_id, +(select cf.id::boolean from community_follower cf where u.id = cf.user_id and ac.id = cf.community_id) as subscribed +from user_ u +cross join all_community ac + +union all + +select +ac.*, +null as user_id, +null as subscribed +from all_community ac +; +-- Post +drop view post_view; +drop view post_aggregates_view; + +-- regen post view +create view post_aggregates_view as +select +p.*, +(select u.banned from user_ u where p.creator_id = u.id) as banned, +(select cb.id::bool from community_user_ban cb where p.creator_id = cb.user_id and p.community_id = cb.community_id) as banned_from_community, +(select actor_id from user_ where p.creator_id = user_.id) as creator_actor_id, +(select local from user_ where p.creator_id = user_.id) as creator_local, +(select name from user_ where p.creator_id = user_.id) as creator_name, +(select avatar from user_ where p.creator_id = user_.id) as creator_avatar, +(select actor_id from community where p.community_id = community.id) as community_actor_id, +(select local from community where p.community_id = community.id) as community_local, +(select name from community where p.community_id = community.id) as community_name, +(select removed from community c where p.community_id = c.id) as community_removed, +(select deleted from community c where p.community_id = c.id) as community_deleted, +(select nsfw from community c where p.community_id = c.id) as community_nsfw, +(select count(*) from comment where comment.post_id = p.id) as number_of_comments, +coalesce(sum(pl.score), 0) as score, +count (case when pl.score = 1 then 1 else null end) as upvotes, +count (case when pl.score = -1 then 1 else null end) as downvotes, +hot_rank(coalesce(sum(pl.score) , 0), + ( + case when (p.published < ('now'::timestamp - '1 month'::interval)) then p.published -- Prevents necro-bumps + else greatest(c.recent_comment_time, p.published) + end + ) +) as hot_rank, +( + case when (p.published < ('now'::timestamp - '1 month'::interval)) then p.published -- Prevents necro-bumps + else greatest(c.recent_comment_time, p.published) + end +) as newest_activity_time +from post p +left join post_like pl on p.id = pl.post_id +left join ( + select post_id, + max(published) as recent_comment_time + from comment + group by 1 +) c on p.id = c.post_id +group by p.id, c.recent_comment_time; + +create materialized view post_aggregates_mview as select * from post_aggregates_view; + +create unique index idx_post_aggregates_mview_id on post_aggregates_mview (id); + +create view post_view as +with all_post as ( + select + pa.* + from post_aggregates_view pa +) +select +ap.*, +u.id as user_id, +coalesce(pl.score, 0) as my_vote, +(select cf.id::bool from community_follower cf where u.id = cf.user_id and cf.community_id = ap.community_id) as subscribed, +(select pr.id::bool from post_read pr where u.id = pr.user_id and pr.post_id = ap.id) as read, +(select ps.id::bool from post_saved ps where u.id = ps.user_id and ps.post_id = ap.id) as saved +from user_ u +cross join all_post ap +left join post_like pl on u.id = pl.user_id and ap.id = pl.post_id + +union all + +select +ap.*, +null as user_id, +null as my_vote, +null as subscribed, +null as read, +null as saved +from all_post ap +; + +create view post_mview as +with all_post as ( + select + pa.* + from post_aggregates_mview pa +) +select +ap.*, +u.id as user_id, +coalesce(pl.score, 0) as my_vote, +(select cf.id::bool from community_follower cf where u.id = cf.user_id and cf.community_id = ap.community_id) as subscribed, +(select pr.id::bool from post_read pr where u.id = pr.user_id and pr.post_id = ap.id) as read, +(select ps.id::bool from post_saved ps where u.id = ps.user_id and ps.post_id = ap.id) as saved +from user_ u +cross join all_post ap +left join post_like pl on u.id = pl.user_id and ap.id = pl.post_id + +union all + +select +ap.*, +null as user_id, +null as my_vote, +null as subscribed, +null as read, +null as saved +from all_post ap +; + +drop trigger refresh_post on post; +create trigger refresh_post +after insert or update or delete or truncate +on post +for each statement +execute procedure refresh_post(); + +create or replace function refresh_post() +returns trigger language plpgsql +as $$ +begin + refresh materialized view concurrently post_aggregates_mview; + refresh materialized view concurrently user_mview; + return null; +end $$; + + +-- User mention, comment, reply +drop view user_mention_view; +drop view comment_view; +drop view comment_aggregates_view; + +-- reply and comment view +create view comment_aggregates_view as +select +c.*, +(select community_id from post p where p.id = c.post_id), +(select co.actor_id from post p, community co where p.id = c.post_id and p.community_id = co.id) as community_actor_id, +(select co.local from post p, community co where p.id = c.post_id and p.community_id = co.id) as community_local, +(select co.name from post p, community co where p.id = c.post_id and p.community_id = co.id) as community_name, +(select u.banned from user_ u where c.creator_id = u.id) as banned, +(select cb.id::bool from community_user_ban cb, post p where c.creator_id = cb.user_id and p.id = c.post_id and p.community_id = cb.community_id) as banned_from_community, +(select actor_id from user_ where c.creator_id = user_.id) as creator_actor_id, +(select local from user_ where c.creator_id = user_.id) as creator_local, +(select name from user_ where c.creator_id = user_.id) as creator_name, +(select avatar from user_ where c.creator_id = user_.id) as creator_avatar, +coalesce(sum(cl.score), 0) as score, +count (case when cl.score = 1 then 1 else null end) as upvotes, +count (case when cl.score = -1 then 1 else null end) as downvotes, +hot_rank(coalesce(sum(cl.score) , 0), c.published) as hot_rank +from comment c +left join comment_like cl on c.id = cl.comment_id +group by c.id; + +create materialized view comment_aggregates_mview as select * from comment_aggregates_view; + +create unique index idx_comment_aggregates_mview_id on comment_aggregates_mview (id); + +create view comment_view as +with all_comment as +( + select + ca.* + from comment_aggregates_view ca +) + +select +ac.*, +u.id as user_id, +coalesce(cl.score, 0) as my_vote, +(select cf.id::boolean from community_follower cf where u.id = cf.user_id and ac.community_id = cf.community_id) as subscribed, +(select cs.id::bool from comment_saved cs where u.id = cs.user_id and cs.comment_id = ac.id) as saved +from user_ u +cross join all_comment ac +left join comment_like cl on u.id = cl.user_id and ac.id = cl.comment_id + +union all + +select + ac.*, + null as user_id, + null as my_vote, + null as subscribed, + null as saved +from all_comment ac +; + +create view comment_mview as +with all_comment as +( + select + ca.* + from comment_aggregates_mview ca +) + +select +ac.*, +u.id as user_id, +coalesce(cl.score, 0) as my_vote, +(select cf.id::boolean from community_follower cf where u.id = cf.user_id and ac.community_id = cf.community_id) as subscribed, +(select cs.id::bool from comment_saved cs where u.id = cs.user_id and cs.comment_id = ac.id) as saved +from user_ u +cross join all_comment ac +left join comment_like cl on u.id = cl.user_id and ac.id = cl.comment_id + +union all + +select + ac.*, + null as user_id, + null as my_vote, + null as subscribed, + null as saved +from all_comment ac +; + +-- Do the reply_view referencing the comment_mview +create view reply_view as +with closereply as ( + select + c2.id, + c2.creator_id as sender_id, + c.creator_id as recipient_id + from comment c + inner join comment c2 on c.id = c2.parent_id + where c2.creator_id != c.creator_id + -- Do union where post is null + union + select + c.id, + c.creator_id as sender_id, + p.creator_id as recipient_id + from comment c, post p + where c.post_id = p.id and c.parent_id is null and c.creator_id != p.creator_id +) +select cv.*, +closereply.recipient_id +from comment_mview cv, closereply +where closereply.id = cv.id +; + +-- user mention +create view user_mention_view as +select + c.id, + um.id as user_mention_id, + c.creator_id, + c.creator_actor_id, + c.creator_local, + c.post_id, + c.parent_id, + c.content, + c.removed, + um.read, + c.published, + c.updated, + c.deleted, + c.community_id, + c.community_actor_id, + c.community_local, + c.community_name, + c.banned, + c.banned_from_community, + c.creator_name, + c.creator_avatar, + c.score, + c.upvotes, + c.downvotes, + c.hot_rank, + c.user_id, + c.my_vote, + c.saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from user_mention um, comment_view c +where um.comment_id = c.id; + + +create view user_mention_mview as +with all_comment as +( + select + ca.* + from comment_aggregates_mview ca +) + +select + ac.id, + um.id as user_mention_id, + ac.creator_id, + ac.creator_actor_id, + ac.creator_local, + ac.post_id, + ac.parent_id, + ac.content, + ac.removed, + um.read, + ac.published, + ac.updated, + ac.deleted, + ac.community_id, + ac.community_actor_id, + ac.community_local, + ac.community_name, + ac.banned, + ac.banned_from_community, + ac.creator_name, + ac.creator_avatar, + ac.score, + ac.upvotes, + ac.downvotes, + ac.hot_rank, + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + (select cs.id::bool from comment_saved cs where u.id = cs.user_id and cs.comment_id = ac.id) as saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from user_ u +cross join all_comment ac +left join comment_like cl on u.id = cl.user_id and ac.id = cl.comment_id +left join user_mention um on um.comment_id = ac.id + +union all + +select + ac.id, + um.id as user_mention_id, + ac.creator_id, + ac.creator_actor_id, + ac.creator_local, + ac.post_id, + ac.parent_id, + ac.content, + ac.removed, + um.read, + ac.published, + ac.updated, + ac.deleted, + ac.community_id, + ac.community_actor_id, + ac.community_local, + ac.community_name, + ac.banned, + ac.banned_from_community, + ac.creator_name, + ac.creator_avatar, + ac.score, + ac.upvotes, + ac.downvotes, + ac.hot_rank, + null as user_id, + null as my_vote, + null as saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from all_comment ac +left join user_mention um on um.comment_id = ac.id +; + diff --git a/server/migrations/2020-06-30-135809_remove_mat_views/up.sql b/server/migrations/2020-06-30-135809_remove_mat_views/up.sql new file mode 100644 index 00000000..bd792a8b --- /dev/null +++ b/server/migrations/2020-06-30-135809_remove_mat_views/up.sql @@ -0,0 +1,939 @@ +-- Drop the mviews +drop view post_mview; +drop materialized view user_mview; +drop view community_mview; +drop materialized view private_message_mview; +drop view user_mention_mview; +drop view reply_view; +drop view comment_mview; +drop materialized view post_aggregates_mview; +drop materialized view community_aggregates_mview; +drop materialized view comment_aggregates_mview; +drop trigger refresh_private_message on private_message; + +-- User +drop view user_view; +create view user_view as +select + u.id, + u.actor_id, + u.name, + u.avatar, + u.email, + u.matrix_user_id, + u.bio, + u.local, + u.admin, + u.banned, + u.show_avatars, + u.send_notifications_to_email, + u.published, + coalesce(pd.posts, 0) as number_of_posts, + coalesce(pd.score, 0) as post_score, + coalesce(cd.comments, 0) as number_of_comments, + coalesce(cd.score, 0) as comment_score +from user_ u +left join ( + select + p.creator_id as creator_id, + count(distinct p.id) as posts, + sum(pl.score) as score + from post p + join post_like pl on p.id = pl.post_id + group by p.creator_id +) pd on u.id = pd.creator_id +left join ( + select + c.creator_id, + count(distinct c.id) as comments, + sum(cl.score) as score + from comment c + join comment_like cl on c.id = cl.comment_id + group by c.creator_id +) cd on u.id = cd.creator_id; + + +create table user_fast as select * from user_view; +alter table user_fast add primary key (id); + +drop trigger refresh_user on user_; + +create trigger refresh_user +after insert or update or delete +on user_ +for each row +execute procedure refresh_user(); + +-- Sample insert +-- insert into user_(name, password_encrypted) values ('test_name', 'bleh'); +-- Sample delete +-- delete from user_ where name like 'test_name'; +-- Sample update +-- update user_ set avatar = 'hai' where name like 'test_name'; +create or replace function refresh_user() +returns trigger language plpgsql +as $$ +begin + IF (TG_OP = 'DELETE') THEN + delete from user_fast where id = OLD.id; + ELSIF (TG_OP = 'UPDATE') THEN + delete from user_fast where id = OLD.id; + insert into user_fast select * from user_view where id = NEW.id; + + -- Refresh post_fast, cause of user info changes + delete from post_aggregates_fast where creator_id = NEW.id; + insert into post_aggregates_fast select * from post_aggregates_view where creator_id = NEW.id; + + delete from comment_aggregates_fast where creator_id = NEW.id; + insert into comment_aggregates_fast select * from comment_aggregates_view where creator_id = NEW.id; + + ELSIF (TG_OP = 'INSERT') THEN + insert into user_fast select * from user_view where id = NEW.id; + END IF; + + return null; +end $$; + +-- Post +-- Redoing the views : Credit eiknat +drop view post_view; +drop view post_aggregates_view; + +create view post_aggregates_view as +select + p.*, + -- creator details + u.actor_id as creator_actor_id, + u."local" as creator_local, + u."name" as creator_name, + u.avatar as creator_avatar, + u.banned as banned, + cb.id::bool as banned_from_community, + -- community details + c.actor_id as community_actor_id, + c."local" as community_local, + c."name" as community_name, + c.removed as community_removed, + c.deleted as community_deleted, + c.nsfw as community_nsfw, + -- post score data/comment count + coalesce(ct.comments, 0) as number_of_comments, + coalesce(pl.score, 0) as score, + coalesce(pl.upvotes, 0) as upvotes, + coalesce(pl.downvotes, 0) as downvotes, + hot_rank( + coalesce(pl.score , 0), ( + case + when (p.published < ('now'::timestamp - '1 month'::interval)) + then p.published + else greatest(ct.recent_comment_time, p.published) + end + ) + ) as hot_rank, + ( + case + when (p.published < ('now'::timestamp - '1 month'::interval)) + then p.published + else greatest(ct.recent_comment_time, p.published) + end + ) as newest_activity_time +from post p +left join user_ u on p.creator_id = u.id +left join community_user_ban cb on p.creator_id = cb.user_id and p.community_id = cb.community_id +left join community c on p.community_id = c.id +left join ( + select + post_id, + count(*) as comments, + max(published) as recent_comment_time + from comment + group by post_id +) ct on ct.post_id = p.id +left join ( + select + post_id, + sum(score) as score, + sum(score) filter (where score = 1) as upvotes, + -sum(score) filter (where score = -1) as downvotes + from post_like + group by post_id +) pl on pl.post_id = p.id +order by p.id; + +create view post_view as +select + pav.*, + us.id as user_id, + us.user_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_read::bool as read, + us.is_saved::bool as saved +from post_aggregates_view pav +cross join lateral ( + select + u.id, + coalesce(cf.community_id, 0) as is_subbed, + coalesce(pr.post_id, 0) as is_read, + coalesce(ps.post_id, 0) as is_saved, + coalesce(pl.score, 0) as user_vote + from user_ u + left join community_user_ban cb on u.id = cb.user_id and cb.community_id = pav.community_id + left join community_follower cf on u.id = cf.user_id and cf.community_id = pav.community_id + left join post_read pr on u.id = pr.user_id and pr.post_id = pav.id + left join post_saved ps on u.id = ps.user_id and ps.post_id = pav.id + left join post_like pl on u.id = pl.user_id and pav.id = pl.post_id +) as us + +union all + +select +pav.*, +null as user_id, +null as my_vote, +null as subscribed, +null as read, +null as saved +from post_aggregates_view pav; + +-- The post fast table +create table post_aggregates_fast as select * from post_aggregates_view; +alter table post_aggregates_fast add primary key (id); + +-- For the hot rank resorting +create index idx_post_aggregates_fast_hot_rank_published on post_aggregates_fast (hot_rank desc, published desc); + +create view post_fast_view as +select + pav.*, + us.id as user_id, + us.user_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_read::bool as read, + us.is_saved::bool as saved +from post_aggregates_fast pav +cross join lateral ( + select + u.id, + coalesce(cf.community_id, 0) as is_subbed, + coalesce(pr.post_id, 0) as is_read, + coalesce(ps.post_id, 0) as is_saved, + coalesce(pl.score, 0) as user_vote + from user_ u + left join community_user_ban cb on u.id = cb.user_id and cb.community_id = pav.community_id + left join community_follower cf on u.id = cf.user_id and cf.community_id = pav.community_id + left join post_read pr on u.id = pr.user_id and pr.post_id = pav.id + left join post_saved ps on u.id = ps.user_id and ps.post_id = pav.id + left join post_like pl on u.id = pl.user_id and pav.id = pl.post_id +) as us + +union all + +select +pav.*, +null as user_id, +null as my_vote, +null as subscribed, +null as read, +null as saved +from post_aggregates_fast pav; + +drop trigger refresh_post on post; + +create trigger refresh_post +after insert or update or delete +on post +for each row +execute procedure refresh_post(); + +-- Sample select +-- select id, name from post_fast_view where name like 'test_post' and user_id is null; +-- Sample insert +-- insert into post(name, creator_id, community_id) values ('test_post', 2, 2); +-- Sample delete +-- delete from post where name like 'test_post'; +-- Sample update +-- update post set community_id = 4 where name like 'test_post'; +create or replace function refresh_post() +returns trigger language plpgsql +as $$ +begin + IF (TG_OP = 'DELETE') THEN + delete from post_aggregates_fast where id = OLD.id; + + -- Update community number of posts + update community_aggregates_fast set number_of_posts = number_of_posts - 1 where id = OLD.community_id; + ELSIF (TG_OP = 'UPDATE') THEN + delete from post_aggregates_fast where id = OLD.id; + insert into post_aggregates_fast select * from post_aggregates_view where id = NEW.id; + ELSIF (TG_OP = 'INSERT') THEN + insert into post_aggregates_fast select * from post_aggregates_view where id = NEW.id; + + -- Update that users number of posts, post score + delete from user_fast where id = NEW.creator_id; + insert into user_fast select * from user_view where id = NEW.creator_id; + + -- Update community number of posts + update community_aggregates_fast set number_of_posts = number_of_posts + 1 where id = NEW.community_id; + + -- Update the hot rank on the post table + -- TODO this might not correctly update it, using a 1 week interval + update post_aggregates_fast as paf + set hot_rank = pav.hot_rank + from post_aggregates_view as pav + where paf.id = pav.id and (pav.published > ('now'::timestamp - '1 week'::interval)); + END IF; + + return null; +end $$; + +-- Community +-- Redoing the views : Credit eiknat +drop view community_moderator_view; +drop view community_follower_view; +drop view community_user_ban_view; +drop view community_view; +drop view community_aggregates_view; + +create view community_aggregates_view as +select + c.id, + c.name, + c.title, + c.description, + c.category_id, + c.creator_id, + c.removed, + c.published, + c.updated, + c.deleted, + c.nsfw, + c.actor_id, + c.local, + c.last_refreshed_at, + u.actor_id as creator_actor_id, + u.local as creator_local, + u.name as creator_name, + u.avatar as creator_avatar, + cat.name as category_name, + coalesce(cf.subs, 0) as number_of_subscribers, + coalesce(cd.posts, 0) as number_of_posts, + coalesce(cd.comments, 0) as number_of_comments, + hot_rank(cf.subs, c.published) as hot_rank +from community c +left join user_ u on c.creator_id = u.id +left join category cat on c.category_id = cat.id +left join ( + select + p.community_id, + count(distinct p.id) as posts, + count(distinct ct.id) as comments + from post p + join comment ct on p.id = ct.post_id + group by p.community_id +) cd on cd.community_id = c.id +left join ( + select + community_id, + count(*) as subs + from community_follower + group by community_id +) cf on cf.community_id = c.id; + +create view community_view as +select + cv.*, + us.user as user_id, + us.is_subbed::bool as subscribed +from community_aggregates_view cv +cross join lateral ( + select + u.id as user, + coalesce(cf.community_id, 0) as is_subbed + from user_ u + left join community_follower cf on u.id = cf.user_id and cf.community_id = cv.id +) as us + +union all + +select + cv.*, + null as user_id, + null as subscribed +from community_aggregates_view cv; + +create view community_moderator_view as +select + cm.*, + u.actor_id as user_actor_id, + u.local as user_local, + u.name as user_name, + u.avatar as avatar, + c.actor_id as community_actor_id, + c.local as community_local, + c.name as community_name +from community_moderator cm +left join user_ u on cm.user_id = u.id +left join community c on cm.community_id = c.id; + +create view community_follower_view as +select + cf.*, + u.actor_id as user_actor_id, + u.local as user_local, + u.name as user_name, + u.avatar as avatar, + c.actor_id as community_actor_id, + c.local as community_local, + c.name as community_name +from community_follower cf +left join user_ u on cf.user_id = u.id +left join community c on cf.community_id = c.id; + +create view community_user_ban_view as +select + cb.*, + u.actor_id as user_actor_id, + u.local as user_local, + u.name as user_name, + u.avatar as avatar, + c.actor_id as community_actor_id, + c.local as community_local, + c.name as community_name +from community_user_ban cb +left join user_ u on cb.user_id = u.id +left join community c on cb.community_id = c.id; + +-- The community fast table + +create table community_aggregates_fast as select * from community_aggregates_view; +alter table community_aggregates_fast add primary key (id); + +create view community_fast_view as +select +ac.*, +u.id as user_id, +(select cf.id::boolean from community_follower cf where u.id = cf.user_id and ac.id = cf.community_id) as subscribed +from user_ u +cross join ( + select + ca.* + from community_aggregates_fast ca +) ac + +union all + +select +caf.*, +null as user_id, +null as subscribed +from community_aggregates_fast caf; + +drop trigger refresh_community on community; + +create trigger refresh_community +after insert or update or delete +on community +for each row +execute procedure refresh_community(); + +-- Sample select +-- select * from community_fast_view where name like 'test_community_name' and user_id is null; +-- Sample insert +-- insert into community(name, title, category_id, creator_id) values ('test_community_name', 'test_community_title', 1, 2); +-- Sample delete +-- delete from community where name like 'test_community_name'; +-- Sample update +-- update community set title = 'test_community_title_2' where name like 'test_community_name'; +create or replace function refresh_community() +returns trigger language plpgsql +as $$ +begin + IF (TG_OP = 'DELETE') THEN + delete from community_aggregates_fast where id = OLD.id; + ELSIF (TG_OP = 'UPDATE') THEN + delete from community_aggregates_fast where id = OLD.id; + insert into community_aggregates_fast select * from community_aggregates_view where id = NEW.id; + + -- Update user view due to owner changes + delete from user_fast where id = NEW.creator_id; + insert into user_fast select * from user_view where id = NEW.creator_id; + + -- Update post view due to community changes + delete from post_aggregates_fast where community_id = NEW.id; + insert into post_aggregates_fast select * from post_aggregates_view where community_id = NEW.id; + + -- TODO make sure this shows up in the users page ? + ELSIF (TG_OP = 'INSERT') THEN + insert into community_aggregates_fast select * from community_aggregates_view where id = NEW.id; + END IF; + + return null; +end $$; + +-- Comment + +drop view user_mention_view; +drop view comment_view; +drop view comment_aggregates_view; + +create view comment_aggregates_view as +select + ct.*, + -- community details + p.community_id, + c.actor_id as community_actor_id, + c."local" as community_local, + c."name" as community_name, + -- creator details + u.banned as banned, + coalesce(cb.id, 0)::bool as banned_from_community, + u.actor_id as creator_actor_id, + u.local as creator_local, + u.name as creator_name, + u.avatar as creator_avatar, + -- score details + coalesce(cl.total, 0) as score, + coalesce(cl.up, 0) as upvotes, + coalesce(cl.down, 0) as downvotes, + hot_rank(coalesce(cl.total, 0), ct.published) as hot_rank +from comment ct +left join post p on ct.post_id = p.id +left join community c on p.community_id = c.id +left join user_ u on ct.creator_id = u.id +left join community_user_ban cb on ct.creator_id = cb.user_id and p.id = ct.post_id and p.community_id = cb.community_id +left join ( + select + l.comment_id as id, + sum(l.score) as total, + count(case when l.score = 1 then 1 else null end) as up, + count(case when l.score = -1 then 1 else null end) as down + from comment_like l + group by comment_id +) as cl on cl.id = ct.id; + +create or replace view comment_view as ( +select + cav.*, + us.user_id as user_id, + us.my_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_saved::bool as saved +from comment_aggregates_view cav +cross join lateral ( + select + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + coalesce(cf.id, 0) as is_subbed, + coalesce(cs.id, 0) as is_saved + from user_ u + left join comment_like cl on u.id = cl.user_id and cav.id = cl.comment_id + left join comment_saved cs on u.id = cs.user_id and cs.comment_id = cav.id + left join community_follower cf on u.id = cf.user_id and cav.community_id = cf.community_id +) as us + +union all + +select + cav.*, + null as user_id, + null as my_vote, + null as subscribed, + null as saved +from comment_aggregates_view cav +); + +-- The fast view +create table comment_aggregates_fast as select * from comment_aggregates_view; +alter table comment_aggregates_fast add primary key (id); + +create view comment_fast_view as +select + cav.*, + us.user_id as user_id, + us.my_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_saved::bool as saved +from comment_aggregates_fast cav +cross join lateral ( + select + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + coalesce(cf.id, 0) as is_subbed, + coalesce(cs.id, 0) as is_saved + from user_ u + left join comment_like cl on u.id = cl.user_id and cav.id = cl.comment_id + left join comment_saved cs on u.id = cs.user_id and cs.comment_id = cav.id + left join community_follower cf on u.id = cf.user_id and cav.community_id = cf.community_id +) as us + +union all + +select + cav.*, + null as user_id, + null as my_vote, + null as subscribed, + null as saved +from comment_aggregates_fast cav; + +-- Do the reply_view referencing the comment_fast_view +create view reply_fast_view as +with closereply as ( + select + c2.id, + c2.creator_id as sender_id, + c.creator_id as recipient_id + from comment c + inner join comment c2 on c.id = c2.parent_id + where c2.creator_id != c.creator_id + -- Do union where post is null + union + select + c.id, + c.creator_id as sender_id, + p.creator_id as recipient_id + from comment c, post p + where c.post_id = p.id and c.parent_id is null and c.creator_id != p.creator_id +) +select cv.*, +closereply.recipient_id +from comment_fast_view cv, closereply +where closereply.id = cv.id +; + +-- user mention +create view user_mention_view as +select + c.id, + um.id as user_mention_id, + c.creator_id, + c.creator_actor_id, + c.creator_local, + c.post_id, + c.parent_id, + c.content, + c.removed, + um.read, + c.published, + c.updated, + c.deleted, + c.community_id, + c.community_actor_id, + c.community_local, + c.community_name, + c.banned, + c.banned_from_community, + c.creator_name, + c.creator_avatar, + c.score, + c.upvotes, + c.downvotes, + c.hot_rank, + c.user_id, + c.my_vote, + c.saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from user_mention um, comment_view c +where um.comment_id = c.id; + +create view user_mention_fast_view as +select + ac.id, + um.id as user_mention_id, + ac.creator_id, + ac.creator_actor_id, + ac.creator_local, + ac.post_id, + ac.parent_id, + ac.content, + ac.removed, + um.read, + ac.published, + ac.updated, + ac.deleted, + ac.community_id, + ac.community_actor_id, + ac.community_local, + ac.community_name, + ac.banned, + ac.banned_from_community, + ac.creator_name, + ac.creator_avatar, + ac.score, + ac.upvotes, + ac.downvotes, + ac.hot_rank, + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + (select cs.id::bool from comment_saved cs where u.id = cs.user_id and cs.comment_id = ac.id) as saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from user_ u +cross join ( + select + ca.* + from comment_aggregates_fast ca +) ac +left join comment_like cl on u.id = cl.user_id and ac.id = cl.comment_id +left join user_mention um on um.comment_id = ac.id + +union all + +select + ac.id, + um.id as user_mention_id, + ac.creator_id, + ac.creator_actor_id, + ac.creator_local, + ac.post_id, + ac.parent_id, + ac.content, + ac.removed, + um.read, + ac.published, + ac.updated, + ac.deleted, + ac.community_id, + ac.community_actor_id, + ac.community_local, + ac.community_name, + ac.banned, + ac.banned_from_community, + ac.creator_name, + ac.creator_avatar, + ac.score, + ac.upvotes, + ac.downvotes, + ac.hot_rank, + null as user_id, + null as my_vote, + null as saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from comment_aggregates_fast ac +left join user_mention um on um.comment_id = ac.id +; + + +drop trigger refresh_comment on comment; + +create trigger refresh_comment +after insert or update or delete +on comment +for each row +execute procedure refresh_comment(); + +-- Sample select +-- select * from comment_fast_view where content = 'test_comment' and user_id is null; +-- Sample insert +-- insert into comment(creator_id, post_id, content) values (2, 2, 'test_comment'); +-- Sample delete +-- delete from comment where content like 'test_comment'; +-- Sample update +-- update comment set removed = true where content like 'test_comment'; +create or replace function refresh_comment() +returns trigger language plpgsql +as $$ +begin + IF (TG_OP = 'DELETE') THEN + delete from comment_aggregates_fast where id = OLD.id; + + -- Update community number of comments + update community_aggregates_fast as caf + set number_of_comments = number_of_comments - 1 + from post as p + where caf.id = p.community_id and p.id = OLD.post_id; + + ELSIF (TG_OP = 'UPDATE') THEN + delete from comment_aggregates_fast where id = OLD.id; + insert into comment_aggregates_fast select * from comment_aggregates_view where id = NEW.id; + ELSIF (TG_OP = 'INSERT') THEN + insert into comment_aggregates_fast select * from comment_aggregates_view where id = NEW.id; + + -- Update user view due to comment count + update user_fast + set number_of_comments = number_of_comments + 1 + where id = NEW.creator_id; + + -- Update post view due to comment count, new comment activity time, but only on new posts + -- TODO this could be done more efficiently + delete from post_aggregates_fast where id = NEW.post_id; + insert into post_aggregates_fast select * from post_aggregates_view where id = NEW.post_id; + + -- Force the hot rank as zero on week-older posts + update post_aggregates_fast as paf + set hot_rank = 0 + where paf.id = NEW.post_id and (paf.published < ('now'::timestamp - '1 week'::interval)); + + -- Update community number of comments + update community_aggregates_fast as caf + set number_of_comments = number_of_comments + 1 + from post as p + where caf.id = p.community_id and p.id = NEW.post_id; + + END IF; + + return null; +end $$; + + +-- post_like +-- select id, score, my_vote from post_fast_view where id = 29 and user_id = 4; +-- Sample insert +-- insert into post_like(user_id, post_id, score) values (4, 29, 1); +-- Sample delete +-- delete from post_like where user_id = 4 and post_id = 29; +-- Sample update +-- update post_like set score = -1 where user_id = 4 and post_id = 29; + +-- TODO test this a LOT +create or replace function refresh_post_like() +returns trigger language plpgsql +as $$ +begin + IF (TG_OP = 'DELETE') THEN + update post_aggregates_fast + set score = case + when (OLD.score = 1) then score - 1 + else score + 1 end, + upvotes = case + when (OLD.score = 1) then upvotes - 1 + else upvotes end, + downvotes = case + when (OLD.score = -1) then downvotes - 1 + else downvotes end + where id = OLD.post_id; + + ELSIF (TG_OP = 'INSERT') THEN + update post_aggregates_fast + set score = case + when (NEW.score = 1) then score + 1 + else score - 1 end, + upvotes = case + when (NEW.score = 1) then upvotes + 1 + else upvotes end, + downvotes = case + when (NEW.score = -1) then downvotes + 1 + else downvotes end + where id = NEW.post_id; + END IF; + + return null; +end $$; + +drop trigger refresh_post_like on post_like; +create trigger refresh_post_like +after insert or delete +on post_like +for each row +execute procedure refresh_post_like(); + +-- comment_like +-- select id, score, my_vote from comment_fast_view where id = 29 and user_id = 4; +-- Sample insert +-- insert into comment_like(user_id, comment_id, post_id, score) values (4, 29, 51, 1); +-- Sample delete +-- delete from comment_like where user_id = 4 and comment_id = 29; +-- Sample update +-- update comment_like set score = -1 where user_id = 4 and comment_id = 29; +create or replace function refresh_comment_like() +returns trigger language plpgsql +as $$ +begin + -- TODO possibly select from comment_fast to get previous scores, instead of re-fetching the views? + IF (TG_OP = 'DELETE') THEN + update comment_aggregates_fast + set score = case + when (OLD.score = 1) then score - 1 + else score + 1 end, + upvotes = case + when (OLD.score = 1) then upvotes - 1 + else upvotes end, + downvotes = case + when (OLD.score = -1) then downvotes - 1 + else downvotes end + where id = OLD.comment_id; + + ELSIF (TG_OP = 'INSERT') THEN + update comment_aggregates_fast + set score = case + when (NEW.score = 1) then score + 1 + else score - 1 end, + upvotes = case + when (NEW.score = 1) then upvotes + 1 + else upvotes end, + downvotes = case + when (NEW.score = -1) then downvotes + 1 + else downvotes end + where id = NEW.comment_id; + END IF; + + return null; +end $$; + +drop trigger refresh_comment_like on comment_like; +create trigger refresh_comment_like +after insert or delete +on comment_like +for each row +execute procedure refresh_comment_like(); + +-- Community user ban + +drop trigger refresh_community_user_ban on community_user_ban; +create trigger refresh_community_user_ban +after insert or delete -- Note this is missing after update +on community_user_ban +for each row +execute procedure refresh_community_user_ban(); + +-- select creator_name, banned_from_community from comment_fast_view where user_id = 4 and content = 'test_before_ban'; +-- select creator_name, banned_from_community, community_id from comment_aggregates_fast where content = 'test_before_ban'; +-- Sample insert +-- insert into comment(creator_id, post_id, content) values (1198, 341, 'test_before_ban'); +-- insert into community_user_ban(community_id, user_id) values (2, 1198); +-- Sample delete +-- delete from community_user_ban where user_id = 1198 and community_id = 2; +-- delete from comment where content = 'test_before_ban'; +-- update comment_aggregates_fast set banned_from_community = false where creator_id = 1198 and community_id = 2; +create or replace function refresh_community_user_ban() +returns trigger language plpgsql +as $$ +begin + -- TODO possibly select from comment_fast to get previous scores, instead of re-fetching the views? + IF (TG_OP = 'DELETE') THEN + update comment_aggregates_fast set banned_from_community = false where creator_id = OLD.user_id and community_id = OLD.community_id; + update post_aggregates_fast set banned_from_community = false where creator_id = OLD.user_id and community_id = OLD.community_id; + ELSIF (TG_OP = 'INSERT') THEN + update comment_aggregates_fast set banned_from_community = true where creator_id = NEW.user_id and community_id = NEW.community_id; + update post_aggregates_fast set banned_from_community = true where creator_id = NEW.user_id and community_id = NEW.community_id; + END IF; + + return null; +end $$; + +-- Community follower + +drop trigger refresh_community_follower on community_follower; +create trigger refresh_community_follower +after insert or delete -- Note this is missing after update +on community_follower +for each row +execute procedure refresh_community_follower(); + +create or replace function refresh_community_follower() +returns trigger language plpgsql +as $$ +begin + IF (TG_OP = 'DELETE') THEN + update community_aggregates_fast set number_of_subscribers = number_of_subscribers - 1 where id = OLD.community_id; + ELSIF (TG_OP = 'INSERT') THEN + update community_aggregates_fast set number_of_subscribers = number_of_subscribers + 1 where id = NEW.community_id; + END IF; + + return null; +end $$; diff --git a/server/migrations/2020-07-08-202609_add_creator_published/down.sql b/server/migrations/2020-07-08-202609_add_creator_published/down.sql new file mode 100644 index 00000000..b8e4452e --- /dev/null +++ b/server/migrations/2020-07-08-202609_add_creator_published/down.sql @@ -0,0 +1,388 @@ +drop view user_mention_view; +drop view reply_fast_view; +drop view comment_fast_view; +drop view comment_view; + +drop view user_mention_fast_view; +drop table comment_aggregates_fast; +drop view comment_aggregates_view; + +create view comment_aggregates_view as +select + ct.*, + -- community details + p.community_id, + c.actor_id as community_actor_id, + c."local" as community_local, + c."name" as community_name, + -- creator details + u.banned as banned, + coalesce(cb.id, 0)::bool as banned_from_community, + u.actor_id as creator_actor_id, + u.local as creator_local, + u.name as creator_name, + u.avatar as creator_avatar, + -- score details + coalesce(cl.total, 0) as score, + coalesce(cl.up, 0) as upvotes, + coalesce(cl.down, 0) as downvotes, + hot_rank(coalesce(cl.total, 0), ct.published) as hot_rank +from comment ct +left join post p on ct.post_id = p.id +left join community c on p.community_id = c.id +left join user_ u on ct.creator_id = u.id +left join community_user_ban cb on ct.creator_id = cb.user_id and p.id = ct.post_id and p.community_id = cb.community_id +left join ( + select + l.comment_id as id, + sum(l.score) as total, + count(case when l.score = 1 then 1 else null end) as up, + count(case when l.score = -1 then 1 else null end) as down + from comment_like l + group by comment_id +) as cl on cl.id = ct.id; + +create or replace view comment_view as ( +select + cav.*, + us.user_id as user_id, + us.my_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_saved::bool as saved +from comment_aggregates_view cav +cross join lateral ( + select + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + coalesce(cf.id, 0) as is_subbed, + coalesce(cs.id, 0) as is_saved + from user_ u + left join comment_like cl on u.id = cl.user_id and cav.id = cl.comment_id + left join comment_saved cs on u.id = cs.user_id and cs.comment_id = cav.id + left join community_follower cf on u.id = cf.user_id and cav.community_id = cf.community_id +) as us + +union all + +select + cav.*, + null as user_id, + null as my_vote, + null as subscribed, + null as saved +from comment_aggregates_view cav +); + +create table comment_aggregates_fast as select * from comment_aggregates_view; +alter table comment_aggregates_fast add primary key (id); + +create view comment_fast_view as +select + cav.*, + us.user_id as user_id, + us.my_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_saved::bool as saved +from comment_aggregates_fast cav +cross join lateral ( + select + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + coalesce(cf.id, 0) as is_subbed, + coalesce(cs.id, 0) as is_saved + from user_ u + left join comment_like cl on u.id = cl.user_id and cav.id = cl.comment_id + left join comment_saved cs on u.id = cs.user_id and cs.comment_id = cav.id + left join community_follower cf on u.id = cf.user_id and cav.community_id = cf.community_id +) as us + +union all + +select + cav.*, + null as user_id, + null as my_vote, + null as subscribed, + null as saved +from comment_aggregates_fast cav; + +create view user_mention_view as +select + c.id, + um.id as user_mention_id, + c.creator_id, + c.creator_actor_id, + c.creator_local, + c.post_id, + c.parent_id, + c.content, + c.removed, + um.read, + c.published, + c.updated, + c.deleted, + c.community_id, + c.community_actor_id, + c.community_local, + c.community_name, + c.banned, + c.banned_from_community, + c.creator_name, + c.creator_avatar, + c.score, + c.upvotes, + c.downvotes, + c.hot_rank, + c.user_id, + c.my_vote, + c.saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from user_mention um, comment_view c +where um.comment_id = c.id; + +create view user_mention_fast_view as +select + ac.id, + um.id as user_mention_id, + ac.creator_id, + ac.creator_actor_id, + ac.creator_local, + ac.post_id, + ac.parent_id, + ac.content, + ac.removed, + um.read, + ac.published, + ac.updated, + ac.deleted, + ac.community_id, + ac.community_actor_id, + ac.community_local, + ac.community_name, + ac.banned, + ac.banned_from_community, + ac.creator_name, + ac.creator_avatar, + ac.score, + ac.upvotes, + ac.downvotes, + ac.hot_rank, + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + (select cs.id::bool from comment_saved cs where u.id = cs.user_id and cs.comment_id = ac.id) as saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from user_ u +cross join ( + select + ca.* + from comment_aggregates_fast ca +) ac +left join comment_like cl on u.id = cl.user_id and ac.id = cl.comment_id +left join user_mention um on um.comment_id = ac.id + +union all + +select + ac.id, + um.id as user_mention_id, + ac.creator_id, + ac.creator_actor_id, + ac.creator_local, + ac.post_id, + ac.parent_id, + ac.content, + ac.removed, + um.read, + ac.published, + ac.updated, + ac.deleted, + ac.community_id, + ac.community_actor_id, + ac.community_local, + ac.community_name, + ac.banned, + ac.banned_from_community, + ac.creator_name, + ac.creator_avatar, + ac.score, + ac.upvotes, + ac.downvotes, + ac.hot_rank, + null as user_id, + null as my_vote, + null as saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from comment_aggregates_fast ac +left join user_mention um on um.comment_id = ac.id +; + +-- Do the reply_view referencing the comment_fast_view +create view reply_fast_view as +with closereply as ( + select + c2.id, + c2.creator_id as sender_id, + c.creator_id as recipient_id + from comment c + inner join comment c2 on c.id = c2.parent_id + where c2.creator_id != c.creator_id + -- Do union where post is null + union + select + c.id, + c.creator_id as sender_id, + p.creator_id as recipient_id + from comment c, post p + where c.post_id = p.id and c.parent_id is null and c.creator_id != p.creator_id +) +select cv.*, +closereply.recipient_id +from comment_fast_view cv, closereply +where closereply.id = cv.id +; + +-- add creator_published to the post view +drop view post_fast_view; +drop table post_aggregates_fast; +drop view post_view; +drop view post_aggregates_view; + +create view post_aggregates_view as +select + p.*, + -- creator details + u.actor_id as creator_actor_id, + u."local" as creator_local, + u."name" as creator_name, + u.avatar as creator_avatar, + u.banned as banned, + cb.id::bool as banned_from_community, + -- community details + c.actor_id as community_actor_id, + c."local" as community_local, + c."name" as community_name, + c.removed as community_removed, + c.deleted as community_deleted, + c.nsfw as community_nsfw, + -- post score data/comment count + coalesce(ct.comments, 0) as number_of_comments, + coalesce(pl.score, 0) as score, + coalesce(pl.upvotes, 0) as upvotes, + coalesce(pl.downvotes, 0) as downvotes, + hot_rank( + coalesce(pl.score , 0), ( + case + when (p.published < ('now'::timestamp - '1 month'::interval)) + then p.published + else greatest(ct.recent_comment_time, p.published) + end + ) + ) as hot_rank, + ( + case + when (p.published < ('now'::timestamp - '1 month'::interval)) + then p.published + else greatest(ct.recent_comment_time, p.published) + end + ) as newest_activity_time +from post p +left join user_ u on p.creator_id = u.id +left join community_user_ban cb on p.creator_id = cb.user_id and p.community_id = cb.community_id +left join community c on p.community_id = c.id +left join ( + select + post_id, + count(*) as comments, + max(published) as recent_comment_time + from comment + group by post_id +) ct on ct.post_id = p.id +left join ( + select + post_id, + sum(score) as score, + sum(score) filter (where score = 1) as upvotes, + -sum(score) filter (where score = -1) as downvotes + from post_like + group by post_id +) pl on pl.post_id = p.id +order by p.id; + +create view post_view as +select + pav.*, + us.id as user_id, + us.user_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_read::bool as read, + us.is_saved::bool as saved +from post_aggregates_view pav +cross join lateral ( + select + u.id, + coalesce(cf.community_id, 0) as is_subbed, + coalesce(pr.post_id, 0) as is_read, + coalesce(ps.post_id, 0) as is_saved, + coalesce(pl.score, 0) as user_vote + from user_ u + left join community_user_ban cb on u.id = cb.user_id and cb.community_id = pav.community_id + left join community_follower cf on u.id = cf.user_id and cf.community_id = pav.community_id + left join post_read pr on u.id = pr.user_id and pr.post_id = pav.id + left join post_saved ps on u.id = ps.user_id and ps.post_id = pav.id + left join post_like pl on u.id = pl.user_id and pav.id = pl.post_id +) as us + +union all + +select +pav.*, +null as user_id, +null as my_vote, +null as subscribed, +null as read, +null as saved +from post_aggregates_view pav; + +create table post_aggregates_fast as select * from post_aggregates_view; +alter table post_aggregates_fast add primary key (id); + +create view post_fast_view as +select + pav.*, + us.id as user_id, + us.user_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_read::bool as read, + us.is_saved::bool as saved +from post_aggregates_fast pav +cross join lateral ( + select + u.id, + coalesce(cf.community_id, 0) as is_subbed, + coalesce(pr.post_id, 0) as is_read, + coalesce(ps.post_id, 0) as is_saved, + coalesce(pl.score, 0) as user_vote + from user_ u + left join community_user_ban cb on u.id = cb.user_id and cb.community_id = pav.community_id + left join community_follower cf on u.id = cf.user_id and cf.community_id = pav.community_id + left join post_read pr on u.id = pr.user_id and pr.post_id = pav.id + left join post_saved ps on u.id = ps.user_id and ps.post_id = pav.id + left join post_like pl on u.id = pl.user_id and pav.id = pl.post_id +) as us + +union all + +select +pav.*, +null as user_id, +null as my_vote, +null as subscribed, +null as read, +null as saved +from post_aggregates_fast pav; \ No newline at end of file diff --git a/server/migrations/2020-07-08-202609_add_creator_published/up.sql b/server/migrations/2020-07-08-202609_add_creator_published/up.sql new file mode 100644 index 00000000..1f2b59ea --- /dev/null +++ b/server/migrations/2020-07-08-202609_add_creator_published/up.sql @@ -0,0 +1,390 @@ +drop view user_mention_view; +drop view reply_fast_view; +drop view comment_fast_view; +drop view comment_view; + +drop view user_mention_fast_view; +drop table comment_aggregates_fast; +drop view comment_aggregates_view; + +create view comment_aggregates_view as +select + ct.*, + -- community details + p.community_id, + c.actor_id as community_actor_id, + c."local" as community_local, + c."name" as community_name, + -- creator details + u.banned as banned, + coalesce(cb.id, 0)::bool as banned_from_community, + u.actor_id as creator_actor_id, + u.local as creator_local, + u.name as creator_name, + u.published as creator_published, + u.avatar as creator_avatar, + -- score details + coalesce(cl.total, 0) as score, + coalesce(cl.up, 0) as upvotes, + coalesce(cl.down, 0) as downvotes, + hot_rank(coalesce(cl.total, 0), ct.published) as hot_rank +from comment ct +left join post p on ct.post_id = p.id +left join community c on p.community_id = c.id +left join user_ u on ct.creator_id = u.id +left join community_user_ban cb on ct.creator_id = cb.user_id and p.id = ct.post_id and p.community_id = cb.community_id +left join ( + select + l.comment_id as id, + sum(l.score) as total, + count(case when l.score = 1 then 1 else null end) as up, + count(case when l.score = -1 then 1 else null end) as down + from comment_like l + group by comment_id +) as cl on cl.id = ct.id; + +create or replace view comment_view as ( +select + cav.*, + us.user_id as user_id, + us.my_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_saved::bool as saved +from comment_aggregates_view cav +cross join lateral ( + select + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + coalesce(cf.id, 0) as is_subbed, + coalesce(cs.id, 0) as is_saved + from user_ u + left join comment_like cl on u.id = cl.user_id and cav.id = cl.comment_id + left join comment_saved cs on u.id = cs.user_id and cs.comment_id = cav.id + left join community_follower cf on u.id = cf.user_id and cav.community_id = cf.community_id +) as us + +union all + +select + cav.*, + null as user_id, + null as my_vote, + null as subscribed, + null as saved +from comment_aggregates_view cav +); + +create table comment_aggregates_fast as select * from comment_aggregates_view; +alter table comment_aggregates_fast add primary key (id); + +create view comment_fast_view as +select + cav.*, + us.user_id as user_id, + us.my_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_saved::bool as saved +from comment_aggregates_fast cav +cross join lateral ( + select + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + coalesce(cf.id, 0) as is_subbed, + coalesce(cs.id, 0) as is_saved + from user_ u + left join comment_like cl on u.id = cl.user_id and cav.id = cl.comment_id + left join comment_saved cs on u.id = cs.user_id and cs.comment_id = cav.id + left join community_follower cf on u.id = cf.user_id and cav.community_id = cf.community_id +) as us + +union all + +select + cav.*, + null as user_id, + null as my_vote, + null as subscribed, + null as saved +from comment_aggregates_fast cav; + +create view user_mention_view as +select + c.id, + um.id as user_mention_id, + c.creator_id, + c.creator_actor_id, + c.creator_local, + c.post_id, + c.parent_id, + c.content, + c.removed, + um.read, + c.published, + c.updated, + c.deleted, + c.community_id, + c.community_actor_id, + c.community_local, + c.community_name, + c.banned, + c.banned_from_community, + c.creator_name, + c.creator_avatar, + c.score, + c.upvotes, + c.downvotes, + c.hot_rank, + c.user_id, + c.my_vote, + c.saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from user_mention um, comment_view c +where um.comment_id = c.id; + +create view user_mention_fast_view as +select + ac.id, + um.id as user_mention_id, + ac.creator_id, + ac.creator_actor_id, + ac.creator_local, + ac.post_id, + ac.parent_id, + ac.content, + ac.removed, + um.read, + ac.published, + ac.updated, + ac.deleted, + ac.community_id, + ac.community_actor_id, + ac.community_local, + ac.community_name, + ac.banned, + ac.banned_from_community, + ac.creator_name, + ac.creator_avatar, + ac.score, + ac.upvotes, + ac.downvotes, + ac.hot_rank, + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + (select cs.id::bool from comment_saved cs where u.id = cs.user_id and cs.comment_id = ac.id) as saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from user_ u +cross join ( + select + ca.* + from comment_aggregates_fast ca +) ac +left join comment_like cl on u.id = cl.user_id and ac.id = cl.comment_id +left join user_mention um on um.comment_id = ac.id + +union all + +select + ac.id, + um.id as user_mention_id, + ac.creator_id, + ac.creator_actor_id, + ac.creator_local, + ac.post_id, + ac.parent_id, + ac.content, + ac.removed, + um.read, + ac.published, + ac.updated, + ac.deleted, + ac.community_id, + ac.community_actor_id, + ac.community_local, + ac.community_name, + ac.banned, + ac.banned_from_community, + ac.creator_name, + ac.creator_avatar, + ac.score, + ac.upvotes, + ac.downvotes, + ac.hot_rank, + null as user_id, + null as my_vote, + null as saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from comment_aggregates_fast ac +left join user_mention um on um.comment_id = ac.id +; + +-- Do the reply_view referencing the comment_fast_view +create view reply_fast_view as +with closereply as ( + select + c2.id, + c2.creator_id as sender_id, + c.creator_id as recipient_id + from comment c + inner join comment c2 on c.id = c2.parent_id + where c2.creator_id != c.creator_id + -- Do union where post is null + union + select + c.id, + c.creator_id as sender_id, + p.creator_id as recipient_id + from comment c, post p + where c.post_id = p.id and c.parent_id is null and c.creator_id != p.creator_id +) +select cv.*, +closereply.recipient_id +from comment_fast_view cv, closereply +where closereply.id = cv.id +; + +-- add creator_published to the post view +drop view post_fast_view; +drop table post_aggregates_fast; +drop view post_view; +drop view post_aggregates_view; + +create view post_aggregates_view as +select + p.*, + -- creator details + u.actor_id as creator_actor_id, + u."local" as creator_local, + u."name" as creator_name, + u.published as creator_published, + u.avatar as creator_avatar, + u.banned as banned, + cb.id::bool as banned_from_community, + -- community details + c.actor_id as community_actor_id, + c."local" as community_local, + c."name" as community_name, + c.removed as community_removed, + c.deleted as community_deleted, + c.nsfw as community_nsfw, + -- post score data/comment count + coalesce(ct.comments, 0) as number_of_comments, + coalesce(pl.score, 0) as score, + coalesce(pl.upvotes, 0) as upvotes, + coalesce(pl.downvotes, 0) as downvotes, + hot_rank( + coalesce(pl.score , 0), ( + case + when (p.published < ('now'::timestamp - '1 month'::interval)) + then p.published + else greatest(ct.recent_comment_time, p.published) + end + ) + ) as hot_rank, + ( + case + when (p.published < ('now'::timestamp - '1 month'::interval)) + then p.published + else greatest(ct.recent_comment_time, p.published) + end + ) as newest_activity_time +from post p +left join user_ u on p.creator_id = u.id +left join community_user_ban cb on p.creator_id = cb.user_id and p.community_id = cb.community_id +left join community c on p.community_id = c.id +left join ( + select + post_id, + count(*) as comments, + max(published) as recent_comment_time + from comment + group by post_id +) ct on ct.post_id = p.id +left join ( + select + post_id, + sum(score) as score, + sum(score) filter (where score = 1) as upvotes, + -sum(score) filter (where score = -1) as downvotes + from post_like + group by post_id +) pl on pl.post_id = p.id +order by p.id; + +create view post_view as +select + pav.*, + us.id as user_id, + us.user_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_read::bool as read, + us.is_saved::bool as saved +from post_aggregates_view pav +cross join lateral ( + select + u.id, + coalesce(cf.community_id, 0) as is_subbed, + coalesce(pr.post_id, 0) as is_read, + coalesce(ps.post_id, 0) as is_saved, + coalesce(pl.score, 0) as user_vote + from user_ u + left join community_user_ban cb on u.id = cb.user_id and cb.community_id = pav.community_id + left join community_follower cf on u.id = cf.user_id and cf.community_id = pav.community_id + left join post_read pr on u.id = pr.user_id and pr.post_id = pav.id + left join post_saved ps on u.id = ps.user_id and ps.post_id = pav.id + left join post_like pl on u.id = pl.user_id and pav.id = pl.post_id +) as us + +union all + +select +pav.*, +null as user_id, +null as my_vote, +null as subscribed, +null as read, +null as saved +from post_aggregates_view pav; + +create table post_aggregates_fast as select * from post_aggregates_view; +alter table post_aggregates_fast add primary key (id); + +create view post_fast_view as +select + pav.*, + us.id as user_id, + us.user_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_read::bool as read, + us.is_saved::bool as saved +from post_aggregates_fast pav +cross join lateral ( + select + u.id, + coalesce(cf.community_id, 0) as is_subbed, + coalesce(pr.post_id, 0) as is_read, + coalesce(ps.post_id, 0) as is_saved, + coalesce(pl.score, 0) as user_vote + from user_ u + left join community_user_ban cb on u.id = cb.user_id and cb.community_id = pav.community_id + left join community_follower cf on u.id = cf.user_id and cf.community_id = pav.community_id + left join post_read pr on u.id = pr.user_id and pr.post_id = pav.id + left join post_saved ps on u.id = ps.user_id and ps.post_id = pav.id + left join post_like pl on u.id = pl.user_id and pav.id = pl.post_id +) as us + +union all + +select +pav.*, +null as user_id, +null as my_vote, +null as subscribed, +null as read, +null as saved +from post_aggregates_fast pav; \ No newline at end of file diff --git a/server/query_testing/generate_explain_reports.sh b/server/query_testing/generate_explain_reports.sh index 6ce7dc42..439b46a7 100755 --- a/server/query_testing/generate_explain_reports.sh +++ b/server/query_testing/generate_explain_reports.sh @@ -1,31 +1,42 @@ #!/bin/bash set -e +# You can import these to http://tatiyants.com/pev/#/plans/new + # Do the views first -echo "explain (analyze, format json) select * from user_mview" > explain.sql -psql -qAt -U lemmy -f explain.sql > user_view.json +echo "explain (analyze, format json) select * from user_fast" > explain.sql +psql -qAt -U lemmy -f explain.sql > user_fast.json -echo "explain (analyze, format json) select * from post_mview where user_id is null order by hot_rank desc, published desc" > explain.sql +echo "explain (analyze, format json) select * from post_view where user_id is null order by hot_rank desc, published desc" > explain.sql psql -qAt -U lemmy -f explain.sql > post_view.json -echo "explain (analyze, format json) select * from comment_mview where user_id is null" > explain.sql +echo "explain (analyze, format json) select * from post_fast_view where user_id is null order by hot_rank desc, published desc" > explain.sql +psql -qAt -U lemmy -f explain.sql > post_fast_view.json + +echo "explain (analyze, format json) select * from comment_view where user_id is null" > explain.sql psql -qAt -U lemmy -f explain.sql > comment_view.json -echo "explain (analyze, format json) select * from community_mview where user_id is null order by hot_rank desc" > explain.sql +echo "explain (analyze, format json) select * from comment_fast_view where user_id is null" > explain.sql +psql -qAt -U lemmy -f explain.sql > comment_fast_view.json + +echo "explain (analyze, format json) select * from community_view where user_id is null order by hot_rank desc" > explain.sql psql -qAt -U lemmy -f explain.sql > community_view.json +echo "explain (analyze, format json) select * from community_fast_view where user_id is null order by hot_rank desc" > explain.sql +psql -qAt -U lemmy -f explain.sql > community_fast_view.json + echo "explain (analyze, format json) select * from site_view limit 1" > explain.sql psql -qAt -U lemmy -f explain.sql > site_view.json -echo "explain (analyze, format json) select * from reply_view where user_id = 34 and recipient_id = 34" > explain.sql -psql -qAt -U lemmy -f explain.sql > reply_view.json +echo "explain (analyze, format json) select * from reply_fast_view where user_id = 34 and recipient_id = 34" > explain.sql +psql -qAt -U lemmy -f explain.sql > reply_fast_view.json echo "explain (analyze, format json) select * from user_mention_view where user_id = 34 and recipient_id = 34" > explain.sql psql -qAt -U lemmy -f explain.sql > user_mention_view.json -echo "explain (analyze, format json) select * from user_mention_mview where user_id = 34 and recipient_id = 34" > explain.sql -psql -qAt -U lemmy -f explain.sql > user_mention_mview.json +echo "explain (analyze, format json) select * from user_mention_fast_view where user_id = 34 and recipient_id = 34" > explain.sql +psql -qAt -U lemmy -f explain.sql > user_mention_fast_view.json grep "Execution Time" *.json diff --git a/server/src/api/claims.rs b/server/src/api/claims.rs new file mode 100644 index 00000000..eec9d1a7 --- /dev/null +++ b/server/src/api/claims.rs @@ -0,0 +1,73 @@ +use diesel::{result::Error, PgConnection}; +use jsonwebtoken::{decode, encode, DecodingKey, EncodingKey, Header, TokenData, Validation}; +use lemmy_db::{user::User_, Crud}; +use lemmy_utils::{is_email_regex, settings::Settings}; +use serde::{Deserialize, Serialize}; + +type Jwt = String; + +#[derive(Debug, Serialize, Deserialize)] +pub struct Claims { + pub id: i32, + pub username: String, + pub iss: String, + pub show_nsfw: bool, + pub theme: String, + pub default_sort_type: i16, + pub default_listing_type: i16, + pub lang: String, + pub avatar: Option, + pub show_avatars: bool, +} + +impl Claims { + pub fn decode(jwt: &str) -> Result, jsonwebtoken::errors::Error> { + let v = Validation { + validate_exp: false, + ..Validation::default() + }; + decode::( + &jwt, + &DecodingKey::from_secret(Settings::get().jwt_secret.as_ref()), + &v, + ) + } + + pub fn jwt(user: User_, hostname: String) -> Jwt { + let my_claims = Claims { + id: user.id, + username: user.name.to_owned(), + iss: hostname, + show_nsfw: user.show_nsfw, + theme: user.theme.to_owned(), + default_sort_type: user.default_sort_type, + default_listing_type: user.default_listing_type, + lang: user.lang.to_owned(), + avatar: user.avatar.to_owned(), + show_avatars: user.show_avatars.to_owned(), + }; + encode( + &Header::default(), + &my_claims, + &EncodingKey::from_secret(Settings::get().jwt_secret.as_ref()), + ) + .unwrap() + } + + // TODO: move these into user? + pub fn find_by_email_or_username( + conn: &PgConnection, + username_or_email: &str, + ) -> Result { + if is_email_regex(username_or_email) { + User_::find_by_email(conn, username_or_email) + } else { + User_::find_by_username(conn, username_or_email) + } + } + + pub fn find_by_jwt(conn: &PgConnection, jwt: &str) -> Result { + let claims: Claims = Claims::decode(&jwt).expect("Invalid token").claims; + User_::read(&conn, claims.id) + } +} diff --git a/server/src/api/comment.rs b/server/src/api/comment.rs index c7406b37..2007542f 100644 --- a/server/src/api/comment.rs +++ b/server/src/api/comment.rs @@ -1,28 +1,7 @@ use crate::{ - api::{APIError, Oper, Perform}, + api::{claims::Claims, APIError, Oper, Perform}, apub::{ApubLikeableType, ApubObjectType}, blocking, - db::{ - comment::*, - comment_view::*, - community_view::*, - moderator::*, - post::*, - site_view::*, - user::*, - user_mention::*, - user_view::*, - Crud, - Likeable, - ListingType, - Saveable, - SortType, - }, - naive_now, - remove_slurs, - scrape_text_for_mentions, - send_email, - settings::Settings, websocket::{ server::{JoinCommunityRoom, SendComment}, UserOperation, @@ -30,6 +9,31 @@ use crate::{ }, DbPool, LemmyError, +}; +use lemmy_db::{ + comment::*, + comment_view::*, + community_view::*, + moderator::*, + naive_now, + post::*, + site_view::*, + user::*, + user_mention::*, + user_view::*, + Crud, + Likeable, + ListingType, + Saveable, + SortType, +}; +use lemmy_utils::{ + make_apub_endpoint, + remove_slurs, + scrape_text_for_mentions, + send_email, + settings::Settings, + EndpointType, MentionData, }; use log::error; @@ -155,7 +159,9 @@ impl Perform for Oper { let inserted_comment_id = inserted_comment.id; let updated_comment: Comment = match blocking(pool, move |conn| { - Comment::update_ap_id(&conn, inserted_comment_id) + let apub_id = + make_apub_endpoint(EndpointType::Comment, &inserted_comment_id.to_string()).to_string(); + Comment::update_ap_id(&conn, inserted_comment_id, apub_id) }) .await? { diff --git a/server/src/api/community.rs b/server/src/api/community.rs index 02071c57..e703dcf4 100644 --- a/server/src/api/community.rs +++ b/server/src/api/community.rs @@ -1,26 +1,24 @@ use super::*; use crate::{ - api::{APIError, Oper, Perform}, - apub::{ - extensions::signatures::generate_actor_keypair, - make_apub_endpoint, - ActorType, - EndpointType, - }, + api::{claims::Claims, APIError, Oper, Perform}, + apub::ActorType, blocking, - db::{Bannable, Crud, Followable, Joinable, SortType}, - is_valid_community_name, - naive_from_unix, - naive_now, - slur_check, - slurs_vec_to_str, websocket::{ server::{JoinCommunityRoom, SendCommunityRoomMessage}, UserOperation, WebsocketInfo, }, DbPool, - LemmyError, +}; +use lemmy_db::{naive_now, Bannable, Crud, Followable, Joinable, SortType}; +use lemmy_utils::{ + generate_actor_keypair, + is_valid_community_name, + make_apub_endpoint, + naive_from_unix, + slur_check, + slurs_vec_to_str, + EndpointType, }; use serde::{Deserialize, Serialize}; use std::str::FromStr; diff --git a/server/src/api/mod.rs b/server/src/api/mod.rs index 6df9909c..bb65815a 100644 --- a/server/src/api/mod.rs +++ b/server/src/api/mod.rs @@ -1,11 +1,8 @@ -use crate::{ - db::{community::*, community_view::*, moderator::*, site::*, user::*, user_view::*}, - websocket::WebsocketInfo, - DbPool, - LemmyError, -}; +use crate::{websocket::WebsocketInfo, DbPool, LemmyError}; use actix_web::client::Client; +use lemmy_db::{community::*, community_view::*, moderator::*, site::*, user::*, user_view::*}; +pub mod claims; pub mod comment; pub mod community; pub mod post; diff --git a/server/src/api/post.rs b/server/src/api/post.rs index 840f1530..cbdb976c 100644 --- a/server/src/api/post.rs +++ b/server/src/api/post.rs @@ -1,27 +1,8 @@ use crate::{ - api::{APIError, Oper, Perform}, + api::{claims::Claims, APIError, Oper, Perform}, apub::{ApubLikeableType, ApubObjectType}, blocking, - db::{ - comment_view::*, - community_view::*, - moderator::*, - post::*, - post_view::*, - site::*, - site_view::*, - user::*, - user_view::*, - Crud, - Likeable, - ListingType, - Saveable, - SortType, - }, fetch_iframely_and_pictrs_data, - naive_now, - slur_check, - slurs_vec_to_str, websocket::{ server::{JoinCommunityRoom, JoinPostRoom, SendPost}, UserOperation, @@ -30,6 +11,24 @@ use crate::{ DbPool, LemmyError, }; +use lemmy_db::{ + comment_view::*, + community_view::*, + moderator::*, + naive_now, + post::*, + post_view::*, + site::*, + site_view::*, + user::*, + user_view::*, + Crud, + Likeable, + ListingType, + Saveable, + SortType, +}; +use lemmy_utils::{is_valid_post_title, make_apub_endpoint, slur_check, slurs_vec_to_str, EndpointType}; use serde::{Deserialize, Serialize}; use std::str::FromStr; @@ -136,6 +135,10 @@ impl Perform for Oper { } } + if !is_valid_post_title(&data.name) { + return Err(APIError::err("invalid_post_title").into()); + } + let user_id = claims.id; // Check for a community ban @@ -157,7 +160,7 @@ impl Perform for Oper { fetch_iframely_and_pictrs_data(&self.client, data.url.to_owned()).await; let post_form = PostForm { - name: data.name.to_owned(), + name: data.name.trim().to_owned(), url: data.url.to_owned(), body: data.body.to_owned(), community_id: data.community_id, @@ -191,11 +194,16 @@ impl Perform for Oper { }; let inserted_post_id = inserted_post.id; - let updated_post = - match blocking(pool, move |conn| Post::update_ap_id(conn, inserted_post_id)).await? { - Ok(post) => post, - Err(_e) => return Err(APIError::err("couldnt_create_post").into()), - }; + let updated_post = match blocking(pool, move |conn| { + let apub_id = + make_apub_endpoint(EndpointType::Post, &inserted_post_id.to_string()).to_string(); + Post::update_ap_id(conn, inserted_post_id, apub_id) + }) + .await? + { + Ok(post) => post, + Err(_e) => return Err(APIError::err("couldnt_create_post").into()), + }; updated_post.send_create(&user, &self.client, pool).await?; @@ -512,6 +520,10 @@ impl Perform for Oper { } } + if !is_valid_post_title(&data.name) { + return Err(APIError::err("invalid_post_title").into()); + } + let claims = match Claims::decode(&data.auth) { Ok(claims) => claims.claims, Err(_e) => return Err(APIError::err("not_logged_in").into()), @@ -561,7 +573,7 @@ impl Perform for Oper { let read_post = blocking(pool, move |conn| Post::read(conn, edit_id)).await??; let post_form = PostForm { - name: data.name.to_owned(), + name: data.name.trim().to_owned(), url: data.url.to_owned(), body: data.body.to_owned(), creator_id: data.creator_id.to_owned(), diff --git a/server/src/api/site.rs b/server/src/api/site.rs index f45561a8..241a80e3 100644 --- a/server/src/api/site.rs +++ b/server/src/api/site.rs @@ -1,31 +1,28 @@ use super::user::Register; use crate::{ - api::{APIError, Oper, Perform}, + api::{claims::Claims, APIError, Oper, Perform}, apub::fetcher::search_by_apub_id, blocking, - db::{ - category::*, - comment_view::*, - community_view::*, - moderator::*, - moderator_views::*, - post_view::*, - site::*, - site_view::*, - user::*, - user_view::*, - Crud, - SearchType, - SortType, - }, - naive_now, - settings::Settings, - slur_check, - slurs_vec_to_str, websocket::{server::SendAllMessage, UserOperation, WebsocketInfo}, DbPool, LemmyError, }; +use lemmy_db::{ + category::*, + comment_view::*, + community_view::*, + moderator::*, + moderator_views::*, + naive_now, + post_view::*, + site::*, + site_view::*, + user_view::*, + Crud, + SearchType, + SortType, +}; +use lemmy_utils::{settings::Settings, slur_check, slurs_vec_to_str}; use log::{debug, info}; use serde::{Deserialize, Serialize}; use std::str::FromStr; diff --git a/server/src/api/user.rs b/server/src/api/user.rs index a4e47e41..9f33843f 100644 --- a/server/src/api/user.rs +++ b/server/src/api/user.rs @@ -1,44 +1,7 @@ use crate::{ - api::{APIError, Oper, Perform}, - apub::{ - extensions::signatures::generate_actor_keypair, - make_apub_endpoint, - ApubObjectType, - EndpointType, - }, + api::{claims::Claims, APIError, Oper, Perform}, + apub::ApubObjectType, blocking, - db::{ - comment::*, - comment_view::*, - community::*, - community_view::*, - moderator::*, - password_reset_request::*, - post::*, - post_view::*, - private_message::*, - private_message_view::*, - site::*, - site_view::*, - user::*, - user_mention::*, - user_mention_view::*, - user_view::*, - Crud, - Followable, - Joinable, - ListingType, - SortType, - }, - generate_random_string, - is_valid_username, - naive_from_unix, - naive_now, - remove_slurs, - send_email, - settings::Settings, - slur_check, - slurs_vec_to_str, websocket::{ server::{JoinUserRoom, SendAllMessage, SendUserRoomMessage}, UserOperation, @@ -48,6 +11,43 @@ use crate::{ LemmyError, }; use bcrypt::verify; +use lemmy_db::{ + comment::*, + comment_view::*, + community::*, + community_view::*, + moderator::*, + naive_now, + password_reset_request::*, + post::*, + post_view::*, + private_message::*, + private_message_view::*, + site::*, + site_view::*, + user::*, + user_mention::*, + user_mention_view::*, + user_view::*, + Crud, + Followable, + Joinable, + ListingType, + SortType, +}; +use lemmy_utils::{ + generate_actor_keypair, + generate_random_string, + is_valid_username, + make_apub_endpoint, + naive_from_unix, + remove_slurs, + send_email, + settings::Settings, + slur_check, + slurs_vec_to_str, + EndpointType, +}; use log::error; use serde::{Deserialize, Serialize}; use std::str::FromStr; @@ -264,7 +264,7 @@ impl Perform for Oper { // Fetch that username / email let username_or_email = data.username_or_email.clone(); let user = match blocking(pool, move |conn| { - User_::find_by_email_or_username(conn, &username_or_email) + Claims::find_by_email_or_username(conn, &username_or_email) }) .await? { @@ -279,7 +279,9 @@ impl Perform for Oper { } // Return the jwt - Ok(LoginResponse { jwt: user.jwt() }) + Ok(LoginResponse { + jwt: Claims::jwt(user, Settings::get().hostname), + }) } } @@ -421,7 +423,7 @@ impl Perform for Oper { // Return the jwt Ok(LoginResponse { - jwt: inserted_user.jwt(), + jwt: Claims::jwt(inserted_user, Settings::get().hostname), }) } } @@ -451,6 +453,11 @@ impl Perform for Oper { None => read_user.email, }; + let avatar = match &data.avatar { + Some(avatar) => Some(avatar.to_owned()), + None => read_user.avatar, + }; + let password_encrypted = match &data.new_password { Some(new_password) => { match &data.new_password_verify { @@ -488,7 +495,7 @@ impl Perform for Oper { name: read_user.name, email, matrix_user_id: data.matrix_user_id.to_owned(), - avatar: data.avatar.to_owned(), + avatar, password_encrypted, preferred_username: read_user.preferred_username, updated: Some(naive_now()), @@ -527,7 +534,7 @@ impl Perform for Oper { // Return the jwt Ok(LoginResponse { - jwt: updated_user.jwt(), + jwt: Claims::jwt(updated_user, Settings::get().hostname), }) } } @@ -678,7 +685,8 @@ impl Perform for Oper { } let added = data.added; - let add_admin = move |conn: &'_ _| User_::add_admin(conn, user_id, added); + let added_user_id = data.user_id; + let add_admin = move |conn: &'_ _| User_::add_admin(conn, added_user_id, added); if blocking(pool, add_admin).await?.is_err() { return Err(APIError::err("couldnt_update_user").into()); } @@ -1149,7 +1157,7 @@ impl Perform for Oper { // Return the jwt Ok(LoginResponse { - jwt: updated_user.jwt(), + jwt: Claims::jwt(updated_user, Settings::get().hostname), }) } } @@ -1207,7 +1215,12 @@ impl Perform for Oper { let inserted_private_message_id = inserted_private_message.id; let updated_private_message = match blocking(pool, move |conn| { - PrivateMessage::update_ap_id(&conn, inserted_private_message_id) + let apub_id = make_apub_endpoint( + EndpointType::PrivateMessage, + &inserted_private_message_id.to_string(), + ) + .to_string(); + PrivateMessage::update_ap_id(&conn, inserted_private_message_id, apub_id) }) .await? { diff --git a/server/src/apub/activities.rs b/server/src/apub/activities.rs index e5dc7045..204a380d 100644 --- a/server/src/apub/activities.rs +++ b/server/src/apub/activities.rs @@ -1,12 +1,18 @@ use crate::{ - apub::{extensions::signatures::sign, is_apub_id_valid, ActorType}, - db::{activity::insert_activity, community::Community, user::User_}, + apub::{ + community::do_announce, + extensions::signatures::sign, + insert_activity, + is_apub_id_valid, + ActorType, + }, request::retry_custom, DbPool, LemmyError, }; use activitystreams::{context, object::properties::ObjectProperties, public, Activity, Base}; use actix_web::client::Client; +use lemmy_db::{community::Community, user::User_}; use log::debug; use serde::Serialize; use std::fmt::Debug; @@ -43,7 +49,7 @@ where // if this is a local community, we need to do an announce from the community instead if community.local { - Community::do_announce(activity, &community, creator, client, pool).await?; + do_announce(activity, &community, creator, client, pool).await?; } else { send_activity(client, &activity, creator, to).await?; } diff --git a/server/src/apub/comment.rs b/server/src/apub/comment.rs index a42a52c2..af3581cb 100644 --- a/server/src/apub/comment.rs +++ b/server/src/apub/comment.rs @@ -17,19 +17,9 @@ use crate::{ ToApub, }, blocking, - convert_datetime, - db::{ - comment::{Comment, CommentForm}, - community::Community, - post::Post, - user::User_, - Crud, - }, routes::DbPoolParam, - scrape_text_for_mentions, DbPool, LemmyError, - MentionData, }; use activitystreams::{ activity::{Create, Delete, Dislike, Like, Remove, Undo, Update}, @@ -40,6 +30,14 @@ use activitystreams::{ use activitystreams_new::object::Tombstone; use actix_web::{body::Body, client::Client, web::Path, HttpResponse}; use itertools::Itertools; +use lemmy_db::{ + comment::{Comment, CommentForm}, + community::Community, + post::Post, + user::User_, + Crud, +}; +use lemmy_utils::{convert_datetime, scrape_text_for_mentions, MentionData}; use log::debug; use serde::Deserialize; @@ -123,7 +121,7 @@ impl FromApub for CommentForm { /// Parse an ActivityPub note received from another instance into a Lemmy comment async fn from_apub( - note: &Note, + note: &mut Note, client: &Client, pool: &DbPool, ) -> Result { diff --git a/server/src/apub/community.rs b/server/src/apub/community.rs index f866511c..8b623e71 100644 --- a/server/src/apub/community.rs +++ b/server/src/apub/community.rs @@ -4,44 +4,48 @@ use crate::{ create_apub_response, create_apub_tombstone_response, create_tombstone, - extensions::{group_extensions::GroupExtension, signatures::PublicKey}, + extensions::group_extensions::GroupExtension, fetcher::get_or_fetch_and_upsert_remote_user, get_shared_inbox, + insert_activity, ActorType, FromApub, GroupExt, ToApub, }, blocking, - convert_datetime, - db::{ - activity::insert_activity, - community::{Community, CommunityForm}, - community_view::{CommunityFollowerView, CommunityModeratorView}, - user::User_, - }, - naive_now, routes::DbPoolParam, DbPool, LemmyError, }; use activitystreams::{ activity::{Accept, Announce, Delete, Remove, Undo}, - actor::{kind::GroupType, properties::ApActorProperties, Group}, - collection::UnorderedCollection, - context, - endpoint::EndpointProperties, - object::properties::ObjectProperties, Activity, Base, BaseBox, }; -use activitystreams_ext::Ext3; -use activitystreams_new::{activity::Follow, object::Tombstone}; +use activitystreams_ext::Ext2; +use activitystreams_new::{ + activity::Follow, + actor::{kind::GroupType, ApActor, Endpoints, Group}, + base::BaseExt, + collection::UnorderedCollection, + context, + object::Tombstone, + prelude::*, + primitives::{XsdAnyUri, XsdDateTime}, +}; use actix_web::{body::Body, client::Client, web, HttpResponse}; use itertools::Itertools; +use lemmy_db::{ + community::{Community, CommunityForm}, + community_view::{CommunityFollowerView, CommunityModeratorView}, + naive_now, + user::User_, +}; +use lemmy_utils::convert_datetime; use serde::{Deserialize, Serialize}; -use std::fmt::Debug; +use std::{fmt::Debug, str::FromStr}; #[derive(Deserialize)] pub struct CommunityQuery { @@ -54,9 +58,6 @@ impl ToApub for Community { // Turn a Lemmy Community into an ActivityPub group that can be sent out over the network. async fn to_apub(&self, pool: &DbPool) -> Result { - let mut group = Group::default(); - let oprops: &mut ObjectProperties = group.as_mut(); - // The attributed to, is an ordered vector with the creator actor_ids first, // then the rest of the moderators // TODO Technically the instance admins can mod the community, but lets @@ -66,36 +67,36 @@ impl ToApub for Community { CommunityModeratorView::for_community(&conn, id) }) .await??; - let moderators = moderators.into_iter().map(|m| m.user_actor_id).collect(); + let moderators: Vec = moderators.into_iter().map(|m| m.user_actor_id).collect(); - oprops - .set_context_xsd_any_uri(context())? - .set_id(self.actor_id.to_owned())? - .set_name_xsd_string(self.name.to_owned())? - .set_published(convert_datetime(self.published))? - .set_many_attributed_to_xsd_any_uris(moderators)?; + let mut group = Group::new(); + group + .set_context(context()) + .set_id(XsdAnyUri::from_str(&self.actor_id)?) + .set_name(self.name.to_owned()) + .set_published(XsdDateTime::from(convert_datetime(self.published))) + .set_many_attributed_tos(moderators); if let Some(u) = self.updated.to_owned() { - oprops.set_updated(convert_datetime(u))?; + group.set_updated(XsdDateTime::from(convert_datetime(u))); } if let Some(d) = self.description.to_owned() { // TODO: this should be html, also add source field with raw markdown // -> same for post.content and others - oprops.set_content_xsd_string(d)?; + group.set_content(d); } - let mut endpoint_props = EndpointProperties::default(); - - endpoint_props.set_shared_inbox(self.get_shared_inbox_url())?; - - let mut actor_props = ApActorProperties::default(); - - actor_props - .set_preferred_username(self.title.to_owned())? - .set_inbox(self.get_inbox_url())? - .set_outbox(self.get_outbox_url())? - .set_endpoints(endpoint_props)? - .set_followers(self.get_followers_url())?; + let mut ap_actor = ApActor::new(self.get_inbox_url().parse()?, group); + ap_actor + .set_preferred_username(self.title.to_owned()) + .set_outbox(self.get_outbox_url().parse()?) + .set_followers(self.get_followers_url().parse()?) + .set_following(self.get_following_url().parse()?) + .set_liked(self.get_liked_url().parse()?) + .set_endpoints(Endpoints { + shared_inbox: Some(self.get_shared_inbox_url().parse()?), + ..Default::default() + }); let nsfw = self.nsfw; let category_id = self.category_id; @@ -104,10 +105,9 @@ impl ToApub for Community { }) .await??; - Ok(Ext3::new( - group, + Ok(Ext2::new( + ap_actor, group_extension, - actor_props, self.get_public_key_ext(), )) } @@ -367,38 +367,52 @@ impl FromApub for CommunityForm { type ApubType = GroupExt; /// Parse an ActivityPub group received from another instance into a Lemmy community. - async fn from_apub(group: &GroupExt, client: &Client, pool: &DbPool) -> Result { - let group_extensions: &GroupExtension = &group.ext_one; - let oprops = &group.inner.object_props; - let aprops = &group.ext_two; - let public_key: &PublicKey = &group.ext_three.public_key; - - let mut creator_and_moderator_uris = oprops.get_many_attributed_to_xsd_any_uris().unwrap(); - let creator_uri = creator_and_moderator_uris.next().unwrap(); + async fn from_apub( + group: &mut GroupExt, + client: &Client, + pool: &DbPool, + ) -> Result { + // TODO: this is probably gonna cause problems cause fetcher:292 also calls take_attributed_to() + let creator_and_moderator_uris = group.clone().take_attributed_to().unwrap(); + let creator_uri = creator_and_moderator_uris + .as_many() + .unwrap() + .iter() + .next() + .unwrap() + .as_xsd_any_uri() + .unwrap(); let creator = get_or_fetch_and_upsert_remote_user(creator_uri.as_str(), client, pool).await?; Ok(CommunityForm { - name: oprops.get_name_xsd_string().unwrap().to_string(), - title: aprops.get_preferred_username().unwrap().to_string(), + name: group + .take_name() + .unwrap() + .as_single_xsd_string() + .unwrap() + .into(), + title: group.inner.take_preferred_username().unwrap(), // TODO: should be parsed as html and tags like