Multiple Lemmy instances on same server

.gitignore

@@ -1,6 +1,7 @@
 ansible/inventory
 ansible/inventory_dev
 ansible/passwords/
+ansible/vars/
 docker/lemmy_mine.hjson
 docker/dev/env_deploy.sh
 build/

ansible/ansible.cfg

@@ -2,4 +2,4 @@
 inventory=inventory
 
 [ssh_connection]
-pipelining = True
+#pipelining = True

ansible/lemmy_dev.yml

@@ -16,6 +16,32 @@
     - setup: # gather facts
 
   tasks:
+  # TODO: this task is running on all hosts at the same time so there is a race condition
+  - name: xxx
+    shell: |
+      mkdir -p "vars/{{ inventory_hostname }}/"
+      if [ ! -f "vars/{{ inventory_hostname }}/port_counter" ]; then
+        if [ -f "vars/max_port_counter" ]; then
+          MAX_PORT=$(cat vars/max_port_counter)
+        else
+         MAX_PORT=8000
+        fi
+        OUR_PORT=$(expr $MAX_PORT + 10)
+        echo $OUR_PORT > "vars/{{ inventory_hostname }}/port_counter"
+        echo $OUR_PORT > "vars/max_port_counter"
+      fi
+      cat "vars/{{ inventory_hostname }}/port_counter"
+    args:
+      executable: /bin/bash
+    delegate_to: localhost
+    register: lemmy_port
+
+  - set_fact: "lemmy_port={{ lemmy_port.stdout_lines[0] }}"
+  - set_fact: "pictshare_port={{ lemmy_port|int + 1 }}"
+  - set_fact: "iframely_port={{ lemmy_port|int + 2 }}"
+  - debug:
+      msg: "lemmy_port={{ lemmy_port }} pictshare_port={{pictshare_port}} iframely_port={{iframely_port}}"
+
   - name: install dependencies
     apt:
       pkg: ['nginx', 'docker-compose', 'docker.io', 'certbot', 'python-certbot-nginx']
@@ -25,25 +51,29 @@
     args:
       creates: '/etc/letsencrypt/live/{{domain}}/privkey.pem'
 
+  # TODO: need to use different path per domain
   - name: create lemmy folder
     file: path={{item.path}} state=directory
     with_items:
-      - { path: '/lemmy/' }
+      - { path: '/lemmy/{{ domain }}/' }
-      - { path: '/lemmy/volumes/' }
+      - { path: '/lemmy/{{ domain }}/volumes/' }
+      - { path: '/var/cache/lemmy/{{ domain }}/' }
 
   - block:
     - name:  add template files
       template: src={{item.src}} dest={{item.dest}} mode={{item.mode}}
       with_items:
-        - { src: 'templates/docker-compose.yml', dest: '/lemmy/docker-compose.yml', mode: '0600' }
+        - { src: 'templates/docker-compose.yml', dest: '/lemmy/{{domain}}/docker-compose.yml', mode: '0600' }
-        - { src: 'templates/nginx.conf', dest: '/etc/nginx/sites-enabled/lemmy.conf', mode: '0644' }
+        - { src: 'templates/nginx.conf', dest: '/etc/nginx/sites-enabled/lemmy-{{ domain }}.conf', mode: '0644' }
-        - { src: '../docker/iframely.config.local.js', dest: '/lemmy/iframely.config.local.js', mode: '0600' }
+        - { src: '../docker/iframely.config.local.js', dest: '/lemmy/{{ domain }}/iframely.config.local.js', mode: '0600' }
 
     - name:  add config file (only during initial setup)
-      template: src='templates/config.hjson' dest='/lemmy/lemmy.hjson' mode='0600' force='no' owner='1000' group='1000'
+      template: src='templates/config.hjson' dest='/lemmy/{{domain}}/lemmy.hjson' mode='0600' force='no' owner='1000' group='1000'
     vars:
-      postgres_password: "{{ lookup('password', 'passwords/{{ inventory_hostname }}/postgres chars=ascii_letters,digits') }}"
+      # TODO: these paths are changed, need to move the files
-      jwt_password: "{{ lookup('password', 'passwords/{{ inventory_hostname }}/jwt chars=ascii_letters,digits') }}"
+      # TODO: not sure what to call the local var folder, its not mentioned in the ansible docs
+      postgres_password: "{{ lookup('password', 'vars/{{ inventory_hostname }}/postgres_password chars=ascii_letters,digits') }}"
+      jwt_password: "{{ lookup('password', 'vars/{{ inventory_hostname }}/jwt_password chars=ascii_letters,digits') }}"
 
   - name: build the dev docker image
     local_action: shell cd .. && sudo docker build . -f docker/dev/Dockerfile -t lemmy:dev
@@ -85,7 +115,7 @@
   # be a problem for testing
   - name: start docker-compose
     docker_compose:
-      project_src: /lemmy/
+      project_src: "/lemmy/{{ domain }}/"
       state: present
       recreate: always
     ignore_errors: yes
@@ -96,6 +126,6 @@
   - name: certbot renewal cronjob
     cron:
       special_time=daily
-      name=certbot-renew-lemmy
+      name=certbot-renew-lemmy-{{ domain }}
       user=root
       job="certbot certonly --nginx -d '{{ domain }}' --deploy-hook 'nginx -s reload'"

ansible/templates/docker-compose.yml

@@ -4,7 +4,7 @@ services:
   lemmy:
     image: {{ lemmy_docker_image }}
     ports:
-      - "127.0.0.1:8536:8536"
+      - "127.0.0.1:{{ lemmy_port }}:8536"
     restart: always
     volumes:
       - ./lemmy.hjson:/config/config.hjson:ro
@@ -26,7 +26,7 @@ services:
   pictshare:
     image: shtripok/pictshare:latest
     ports:
-      - "127.0.0.1:8537:80"
+      - "127.0.0.1:{{ pictshare_port }}:80"
     volumes:
       - ./volumes/pictshare:/usr/share/nginx/html/data
     restart: always
@@ -34,7 +34,7 @@ services:
   iframely:
     image: dogbin/iframely:latest
     ports:
-      - "127.0.0.1:8061:80"
+      - "127.0.0.1:{{ iframely_port }}:80"
     volumes:
       - ./iframely.config.local.js:/iframely/config.local.js:ro
     restart: always

ansible/templates/nginx.conf

@@ -1,4 +1,4 @@
-proxy_cache_path /var/cache/lemmy_frontend levels=1:2 keys_zone=lemmy_frontend_cache:10m max_size=100m                 use_temp_path=off;
+proxy_cache_path /var/cache/lemmy/{{ domain }} levels=1:2 keys_zone=lemmy_frontend_cache_{{ domain }}:10m max_size=100m use_temp_path=off;
 
 server {
     listen 80;
@@ -52,7 +52,7 @@ server {
     client_max_body_size 50M;
 
     location / {
-        proxy_pass http://0.0.0.0:8536;
+        proxy_pass http://0.0.0.0:{{ lemmy_port }};
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header Host $host;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -63,7 +63,7 @@ server {
         proxy_set_header Connection "upgrade";
 
         # Proxy Cache
-        proxy_cache             lemmy_frontend_cache;
+        proxy_cache             lemmy_frontend_cache_{{ domain }};
         proxy_cache_use_stale   error timeout http_500 http_502 http_503 http_504;
         proxy_cache_revalidate  on;
         proxy_cache_lock        on;
@@ -71,7 +71,7 @@ server {
     }    
 
     location /pictshare/ {
-      proxy_pass http://0.0.0.0:8537/;
+      proxy_pass http://0.0.0.0:{{ pictshare_port }}/;
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header Host $host;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -82,7 +82,7 @@ server {
     }
 
     location /iframely/ {
-      proxy_pass http://0.0.0.0:8061/;
+      proxy_pass http://0.0.0.0:{{ iframely_port }}/;
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header Host $host;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -98,6 +98,6 @@ map $remote_addr $remote_addr_anon {
   ::1                         $remote_addr;
   default                     0.0.0.0;
 }
-log_format main '$remote_addr_anon - $remote_user [$time_local] "$request" '
+log_format main_{{ domain }} '$remote_addr_anon - $remote_user [$time_local] "$request" '
 '$status $body_bytes_sent "$http_referer" "$http_user_agent"';
-access_log /var/log/nginx/access.log main;
+access_log /var/log/nginx/access.log main_{{ domain }};