grishka/lemmy
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Combine prod and dev docker setups using build-arg (#2739)

Browse source 
* Combine prod and dev docker setups using build-arg

- Fixes #2603

* Dont use cache for release build.
This commit is contained in:
Dessalines 2023-02-17 01:29:45 -05:00 committed by GitHub
parent 36aaa50644
commit 993489a80f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 54 additions and 363 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.drone.yml
View file

@ -120,7 +120,8 @@ steps:
- name: nightly build
image: plugins/docker
settings:
dockerfile: docker/prod/Dockerfile
dockerfile: docker/Dockerfile
build_args: RUST_RELEASE_MODE=release
username:
from_secret: docker_username
password:
@ -136,7 +137,8 @@ steps:
- name: publish release docker image
image: plugins/docker
settings:
dockerfile: docker/prod/Dockerfile
dockerfile: docker/Dockerfile
build_args: RUST_RELEASE_MODE=release
username:
from_secret: docker_username
password:
@ -285,7 +287,7 @@ steps:
- name: publish release docker image
image: plugins/docker
settings:
dockerfile: docker/prod/Dockerfile.arm
dockerfile: docker/Dockerfile.arm
username:
from_secret: docker_username
password:

37
docker/Dockerfile Normal file
View file

@ -0,0 +1,37 @@
FROM clux/muslrust:1.67.0 as builder
WORKDIR /app
ARG CARGO_BUILD_TARGET=x86_64-unknown-linux-musl
# This can be set to release using --build-arg
ARG RUST_RELEASE_MODE="debug"
COPY . .
# Build the project
RUN echo "pub const VERSION: &str = \"$(git describe --tag)\";" > "crates/utils/src/version.rs"
# Debug mode build
RUN --mount=type=cache,target=/app/target \
if [ "$RUST_RELEASE_MODE" = "debug" ] ; then \
cargo build --target ${CARGO_BUILD_TARGET} \
&& cp ./target/$CARGO_BUILD_TARGET/$RUST_RELEASE_MODE/lemmy_server /app/lemmy_server; \
fi
# Release mode build
RUN \
if [ "$RUST_RELEASE_MODE" = "release" ] ; then \
cargo build --target ${CARGO_BUILD_TARGET} --release \
&& cp ./target/$CARGO_BUILD_TARGET/$RUST_RELEASE_MODE/lemmy_server /app/lemmy_server; \
fi
# The alpine runner
FROM alpine:3 as lemmy
# Install libpq for postgres
RUN apk add libpq
# Copy resources
COPY --from=builder /app/lemmy_server /app/lemmy
EXPOSE 8536
CMD ["/app/lemmy"]

0
docker/prod/Dockerfile.arm → docker/Dockerfile.arm
View file

44
docker/dev/Dockerfile
View file

@ -1,44 +0,0 @@
ARG RUST_BUILDER_IMAGE=clux/muslrust:1.67.0
FROM $RUST_BUILDER_IMAGE as chef
USER root
RUN cargo install cargo-chef
WORKDIR /app
# Cargo chef plan
FROM chef as planner
ENV RUSTFLAGS="--cfg tokio_unstable"
# Copy dirs
COPY . .
RUN cargo chef prepare --recipe-path recipe.json
FROM chef as builder
ARG CARGO_BUILD_TARGET=x86_64-unknown-linux-musl
ARG RUSTRELEASEDIR="debug"
ENV RUSTFLAGS="--cfg tokio_unstable"
COPY --from=planner /app/recipe.json ./recipe.json
RUN cargo chef cook --recipe-path recipe.json --target ${CARGO_BUILD_TARGET}
# Copy the rest of the dirs
COPY . .
# Build the project
RUN echo "pub const VERSION: &str = \"$(git describe --tag)\";" > "crates/utils/src/version.rs"
RUN cargo build --target ${CARGO_BUILD_TARGET}
RUN cp ./target/$CARGO_BUILD_TARGET/$RUSTRELEASEDIR/lemmy_server /app/lemmy_server
# The alpine runner
FROM alpine:3 as lemmy
# Install libpq for postgres
RUN apk add libpq
# Copy resources
COPY --from=builder /app/lemmy_server /app/lemmy
EXPOSE 8536
CMD ["/app/lemmy"]

6
docker/dev/docker-compose.yml → docker/docker-compose.yml
View file

@ -30,8 +30,10 @@ services:
# use this to build your local lemmy server image for development
# run docker compose up --build
build:
context: ../..
dockerfile: docker/dev/Dockerfile
context: ../
dockerfile: docker/Dockerfile
# args:
# RUST_RELEASE_MODE: release
# this hostname is used in nginx reverse proxy and also for lemmy ui to connect to the backend, do not change
hostname: lemmy
networks:

2
docker/docker_db_backup.sh
View file

@ -1,4 +1,2 @@
#!/bin/bash
pushd dev
docker-compose exec postgres pg_dumpall -c -U lemmy > dump_`date +%Y-%m-%d"_"%H_%M_%S`.sql
popd

0
docker/dev/docker_update.sh → docker/docker_update.sh
View file

2
docker/federation/start-local-instances.bash
View file

@ -3,7 +3,7 @@ set -e
sudo docker-compose down
sudo docker build ../../ --file ../dev/Dockerfile -t lemmy-federation:latest
sudo docker build ../../ --file ../Dockerfile -t lemmy-federation:latest
for Item in alpha beta gamma delta epsilon ; do
sudo mkdir -p volumes/pictrs_$Item

2
docker/dev/lemmy.hjson → docker/lemmy.hjson
View file

@ -23,5 +23,5 @@
# api_key: "API_KEY"
}
opentelemetry_url: "http://otel:4137"
#opentelemetry_url: "http://otel:4137"
}

0
docker/dev/nginx.conf → docker/nginx.conf
View file

0
docker/dev/otel.yml → docker/otel.yml
View file

26
docker/prod/Dockerfile
View file

@ -1,26 +0,0 @@
# Build the project
FROM clux/muslrust:1.67.0 as builder
ARG CARGO_BUILD_TARGET=x86_64-unknown-linux-musl
ARG RUSTRELEASEDIR="release"
WORKDIR /app
COPY ./ ./
RUN echo "pub const VERSION: &str = \"$(git describe --tag)\";" > "crates/utils/src/version.rs"
RUN cargo build --release
RUN cp ./target/$CARGO_BUILD_TARGET/$RUSTRELEASEDIR/lemmy_server /app/lemmy_server
# The alpine runner
FROM alpine:3 as lemmy
# Install libpq for postgres
RUN apk update && apk add libpq
# Copy resources
COPY --from=builder /app/lemmy_server /app/lemmy
EXPOSE 8536
CMD ["/app/lemmy"]

86
docker/prod/docker-compose.yml
View file

@ -1,86 +0,0 @@
version: "3.3"
networks:
# communication to web and clients
lemmyexternalproxy:
# communication between lemmy services
lemmyinternal:
driver: bridge
internal: true
services:
proxy:
image: nginx:1-alpine
networks:
- lemmyinternal
- lemmyexternalproxy
ports:
# only ports facing any connection from outside
- 80:80
- 443:443
volumes:
- ./nginx.conf:/etc/nginx/nginx.conf:ro
# setup your certbot and letsencrypt config
- ./certbot:/var/www/certbot
- ./letsencrypt:/etc/letsencrypt/live
restart: always
depends_on:
- pictrs
- lemmy-ui
lemmy:
image: dessalines/lemmy:0.17.1
hostname: lemmy
networks:
- lemmyinternal
restart: always
environment:
- RUST_LOG="warn,lemmy_server=info,lemmy_api=info,lemmy_api_common=info,lemmy_api_crud=info,lemmy_apub=info,lemmy_db_schema=info,lemmy_db_views=info,lemmy_db_views_actor=info,lemmy_db_views_moderator=info,lemmy_routes=info,lemmy_utils=info,lemmy_websocket=info"
volumes:
- ./lemmy.hjson:/config/config.hjson
depends_on:
- postgres
- pictrs
lemmy-ui:
image: dessalines/lemmy-ui:0.17.1
networks:
- lemmyinternal
environment:
# this needs to match the hostname defined in the lemmy service
- LEMMY_INTERNAL_HOST=lemmy:8536
# set the outside hostname here
- LEMMY_EXTERNAL_HOST=localhost:1236
- LEMMY_HTTPS=true
depends_on:
- lemmy
restart: always
pictrs:
image: asonix/pictrs:0.3.1
# this needs to match the pictrs url in lemmy.hjson
hostname: pictrs
# we can set options to pictrs like this, here we set max. image size and forced format for conversion
# entrypoint: /sbin/tini -- /usr/local/bin/pict-rs -p /mnt -m 4 --image-format webp
networks:
- lemmyinternal
environment:
- PICTRS__API_KEY=API_KEY
user: 991:991
volumes:
- ./volumes/pictrs:/mnt
restart: always
postgres:
image: postgres:15-alpine
# this needs to match the database host in lemmy.hson
hostname: postgres
networks:
- lemmyinternal
environment:
- POSTGRES_USER=lemmy
- POSTGRES_PASSWORD=password
- POSTGRES_DB=lemmy
volumes:
- ./volumes/postgres:/var/lib/postgresql/data
restart: always

43
docker/prod/lemmy.hjson
View file

@ -1,43 +0,0 @@
{
# for more info about the config, check out the documentation
# https://join-lemmy.org/docs/en/administration/configuration.html
# only few config options are covered in this example config
setup: {
# username for the admin user
admin_username: "lemmy"
# password for the admin user
admin_password: "lemmylemmy"
# name of the site (can be changed later)
site_name: "mylemmyinstance"
}
# the domain name of your instance (eg "lemmy.ml")
hostname: "mydomain.ml"
# address where lemmy should listen for incoming requests
bind: "0.0.0.0"
# port where lemmy should listen for incoming requests
port: 8536
# Whether the site is available over TLS. Needs to be true for federation to work.
tls_enabled: true
# pictrs host
pictrs_url: "http://pictrs:8080"
# settings related to the postgresql database
database: {
# name of the postgres database for lemmy
database: "lemmy"
# username to connect to postgres
user: "lemmy"
# password to connect to postgres
password: "password"
# host where postgres is running
host: "postgres"
# port where postgres can be accessed
port: 5432
# maximum number of active sql connections
pool_size: 5
}
}

147
docker/prod/nginx.conf
View file

@ -1,147 +0,0 @@
# nginx example config
# replace {{yourdomain}} and review the certbot/letsencrypt config
limit_req_zone $binary_remote_addr zone={{yourdomain}}_ratelimit:10m rate=1r/s;
upstream lemmy {
# this needs to map to the lemmy (server) docker service hostname
server "lemmy:8536";
}
upstream lemmy-ui {
# this needs to map to the lemmy-ui docker service hostname
server "lemmy-ui:1234";
}
server {
# allow letsencrypt challenge
# redirect everything else to 443
listen 80;
listen [::]:80;
server_name {{yourdomain}};
location /.well-known/acme-challenge/ {
root /var/www/certbot;
}
location / {
return 301 https://$host$request_uri;
}
}
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name {{yourdomain}};
ssl_certificate /etc/letsencrypt/live/{{yourdomain}}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/{{yourdomain}}/privkey.pem;
# Various TLS hardening settings
# https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html
ssl_protocols TLSv1.2 TLSv1.3;
ssl_prefer_server_ciphers on;
ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
ssl_session_timeout 10m;
ssl_session_cache shared:SSL:10m;
ssl_session_tickets on;
ssl_stapling on;
ssl_stapling_verify on;
# Hide nginx version
server_tokens off;
# Enable compression for JS/CSS/HTML bundle, for improved client load times.
# It might be nice to compress JSON, but leaving that out to protect against potential
# compression+encryption information leak attacks like BREACH.
gzip on;
gzip_types text/css application/javascript image/svg+xml;
gzip_vary on;
# Only connect to this site via HTTPS for the two years
add_header Strict-Transport-Security "max-age=63072000";
# Various content security headers
add_header Referrer-Policy "same-origin";
add_header X-Content-Type-Options "nosniff";
add_header X-Frame-Options "DENY";
add_header X-XSS-Protection "1; mode=block";
# Upload limit for pictrs
client_max_body_size 20M;
# frontend
location / {
# distinguish between ui requests and backend
# don't change lemmy-ui or lemmy here, they refer to the upstream definitions on top
set $proxpass "http://lemmy-ui";
if ($http_accept = "application/activity+json") {
set $proxpass "http://lemmy";
}
if ($http_accept = "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"") {
set $proxpass "http://lemmy";
}
if ($request_method = POST) {
set $proxpass "http://lemmy";
}
proxy_pass $proxpass;
rewrite ^(.+)/+$ $1 permanent;
# Send actual client IP upstream
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
# backend
location ~ ^/(api|feeds|nodeinfo|.well-known) {
proxy_pass "http://lemmy";
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
# Rate limit
limit_req zone={{yourdomain}}_ratelimit burst=30 nodelay;
# Add IP forwarding headers
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
# pictrs only - for adding browser cache control.
location ~ ^/(pictrs) {
# allow browser cache, images never update, we can apply long term cache
expires 120d;
add_header Pragma "public";
add_header Cache-Control "public";
proxy_pass "http://lemmy";
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
# Rate limit
limit_req zone={{yourdomain}}_ratelimit burst=30 nodelay;
# Add IP forwarding headers
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
# Redirect pictshare images to pictrs
location ~ /pictshare/(.*)$ {
return 301 /pictrs/image/$1;
}
}
# Anonymize IP addresses
# https://www.supertechcrew.com/anonymizing-logs-nginx-apache/
map $remote_addr $remote_addr_anon {
~(?P<ip>\d+\.\d+\.\d+)\. $ip.0;
~(?P<ip>[^:]+:[^:]+): $ip::;
127.0.0.1 $remote_addr;
::1 $remote_addr;
default 0.0.0.0;
}
access_log /var/log/nginx/access.log combined;

2
docker/dev/test_deploy.sh → docker/test_deploy.sh
View file

@ -5,7 +5,7 @@ export COMPOSE_DOCKER_CLI_BUILD=1
export DOCKER_BUILDKIT=1
# Rebuilding dev docker
sudo docker build ../../ -f . -t "dessalines/lemmy:dev"
sudo docker build ../ -f . -t "dessalines/lemmy:dev"
sudo docker push "dessalines/lemmy:dev"
# Run the playbook

10
scripts/release.sh
View file

@ -9,13 +9,11 @@ third_semver=$(echo $new_tag | cut -d "." -f 3)
# The ansible and docker installs should only update for non release-candidates
# IE, when the third semver is a number, not '2-rc'
if [ ! -z "${third_semver##*[!0-9]*}" ]; then
pushd ../docker/prod/
sed -i "s/dessalines\/lemmy:.*/dessalines\/lemmy:$new_tag/" ../prod/docker-compose.yml
sed -i "s/dessalines\/lemmy-ui:.*/dessalines\/lemmy-ui:$new_tag/" ../prod/docker-compose.yml
sed -i "s/dessalines\/lemmy-ui:.*/dessalines\/lemmy-ui:$new_tag/" ../dev/docker-compose.yml
pushd ../docker
sed -i "s/dessalines\/lemmy:.*/dessalines\/lemmy:$new_tag/" ../docker-compose.yml
sed -i "s/dessalines\/lemmy-ui:.*/dessalines\/lemmy-ui:$new_tag/" ../docker-compose.yml
sed -i "s/dessalines\/lemmy-ui:.*/dessalines\/lemmy-ui:$new_tag/" ../federation/docker-compose.yml
git add ../prod/docker-compose.yml
git add ../dev/docker-compose.yml
git add ../docker-compose.yml
git add ../federation/docker-compose.yml
popd

2
scripts/restore_db.sh
View file

@ -1,5 +1,5 @@
#!/bin/bash
psql -U lemmy -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;"
cat docker/dev/lemmy_dump_2021-01-29_16_13_40.sqldump | psql -U lemmy
cat docker/lemmy_dump_2021-01-29_16_13_40.sqldump | psql -U lemmy
psql -U lemmy -c "alter user lemmy with password 'password'"