Make docker-compose more clear and explicit (#2469)

* Make docker-compose more clear, starting with dev

Explicit networks, not opining ports when not necessary: Confusing for users. All requests go through nginx, so there is no need to open ports directly on pictrs and other services for example. I wasn't sure about jaeger and otel, maybe that can be changed somewhat too. 

Also adding comments and things that are relevant for a prod setup too.

* Update with comments

* Providing dedicated nginx and lemmy conf

* Fix config key for pictrs.

* Fix config key for pictrs.

* updates on naming

* Adding docker compose build info

* Revert to more complete lemmy config

* Test and fix build paths

* Fix otel config

* Delete dev file

Co-authored-by: Dessalines <dessalines@users.noreply.github.com>

docker/dev/docker-compose.yml

@@ -1,47 +1,100 @@
-version: '3.3'
+version: "3.3"
+
+networks:
+  # communication to web and clients
+  lemmyexternalproxy:
+  # communication between lemmy services
+  lemmyinternal:
+    driver: bridge
+    internal: true
 
 services:
-  nginx:
+  proxy:
     image: nginx:1-alpine
+    networks:
+      - lemmyinternal
+      - lemmyexternalproxy
     ports:
+    # actual and only port facing any connection from outside 
       - "1236:1236"
     volumes:
-      - ./nginx.conf:/etc/nginx/nginx.conf
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
     restart: always
     depends_on:
       - pictrs
       - lemmy-ui
 
   lemmy:
-    image: lemmy-dev:latest
+    image: dessalines/lemmy:dev
-    ports:
+    # use this to build your local lemmy server image for development
-      - "8536:8536"
+    # run docker compose up --build
-      - "6669:6669"
+    # build: 
+    #   context: ../..
+    #   dockerfile: docker/dev/Dockerfile
+    # this hostname is used in nginx reverse proxy and also for lemmy ui to connect to the backend, do not change
+    hostname: lemmy
+    networks:
+      - lemmyinternal
     restart: always
     environment:
       - RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
+      - RUST_BACKTRACE=full
     volumes:
-      - ../lemmy.hjson:/config/config.hjson
+      - ./lemmy.hjson:/config/config.hjson
-    depends_on: 
+    depends_on:
-      - pictrs
       - postgres
-      - otel
+      - pictrs
 
   lemmy-ui:
-    image: dessalines/lemmy-ui:0.16.5
+    image: dessalines/lemmy-ui:dev
-    restart: always
+    # use this to build your local lemmy ui image for development
+    # run docker compose up --build
+    # assuming lemmy-ui is cloned besides lemmy directory
+    # build: 
+    #  context: ../../../lemmy-ui
+    #  dockerfile: Dockerfile  
+    networks:
+      - lemmyinternal
     environment:
-      - LEMMY_INTERNAL_HOST=lemmy:8536
+      # this needs to match the hostname defined in the lemmy service
-      - LEMMY_EXTERNAL_HOST=localhost:1234
+      - LEMMY_UI_LEMMY_INTERNAL_HOST=lemmy:8536
+      # set the outside hostname here
+      - LEMMY_UI_LEMMY_EXTERNAL_HOST=localhost:1236
       - LEMMY_HTTPS=false
       - LEMMY_UI_DEBUG=true
-    depends_on: 
+    depends_on:
       - lemmy
+    restart: always
+
+  pictrs:
+    image: asonix/pictrs:0.3.1
+    # this needs to match the pictrs url in lemmy.hjson
+    hostname: pictrs
+    # we can set options to pictrs like this, here we set max. image size and forced format for conversion
+    # entrypoint: /sbin/tini -- /usr/local/bin/pict-rs -p /mnt -m 4 --image-format webp
+    networks:
+      - lemmyinternal
+    environment:
+      - PICTRS_OPENTELEMETRY_URL=http://otel:4137
+      - PICTRS__API_KEY=API_KEY
+      - RUST_LOG=debug
+      - RUST_BACKTRACE=full
+    user: 991:991
+    volumes:
+      - ./volumes/pictrs:/mnt
+    restart: always
 
   postgres:
     image: postgres:14-alpine
+    # this needs to match the database host in lemmy.hson
+    hostname: postgres
+    command: ["postgres", "-c", "session_preload_libraries=auto_explain", "-c", "auto_explain.log_min_duration=5ms", "-c", "auto_explain.log_analyze=true"]
+    networks:
+      - lemmyinternal
+    # adding the external facing network to allow direct db access for devs
+      - lemmyexternalproxy
     ports:
-      # use a different port so it doesnt conflict with postgres running on the host
+    # use a different port so it doesnt conflict with potential postgres db running on the host
       - "5433:5432"
     environment:
       - POSTGRES_USER=lemmy
@@ -50,40 +103,30 @@ services:
     volumes:
       - ./volumes/postgres:/var/lib/postgresql/data
     restart: always
-    command: ["postgres", "-c", "session_preload_libraries=auto_explain", "-c", "auto_explain.log_min_duration=5ms", "-c", "auto_explain.log_analyze=true"]
-
-  pictrs:
-    image: asonix/pictrs:0.3.1
-    user: 991:991
-    environment:
-      - PICTRS_OPENTELEMETRY_URL=http://otel:4137
-      - PICTRS__API_KEY=API_KEY
-    ports:
-      - "6670:6669"
-      - "8080:8080"
-    volumes:
-      - ./volumes/pictrs:/mnt
-    restart: always
-    depends_on:
-      - otel
 
   otel:
     image: otel/opentelemetry-collector:latest
     command: --config otel-local-config.yaml
+    networks:
+      - lemmyinternal
+      - lemmyexternalproxy
     ports:
       - "4317:4317"
     volumes:
       - type: bind
         source: ./otel.yml
         target: /otel-local-config.yaml
-    restart: always
+    restart: unless-stopped
     depends_on:
       - jaeger
-
+  
   jaeger:
     image: jaegertracing/all-in-one:1
+    networks:
+      - lemmyinternal
+      - lemmyexternalproxy
     ports:
       - "14250:14250"
       # To view traces, visit http://localhost:16686
       - "16686:16686"
-    restart: always
+    restart: unless-stopped

docker/dev/lemmy.hjson

@@ -0,0 +1,25 @@
+{
+  # for more info about the config, check out the documentation
+  # https://join-lemmy.org/docs/en/administration/configuration.html
+
+  # This is a minimal lemmy config for the dev / main branch. Do not use for a 
+  # release / stable version.
+
+  setup: {
+    # username for the admin user
+    admin_username: "lemmy"
+    # password for the admin user
+    admin_password: "lemmylemmy"
+    # name of the site (can be changed later)
+    site_name: "lemmy-dev"
+  }
+
+  opentelemetry_url: "http://otel:4137"
+
+  # the domain name of your instance (eg "lemmy.ml")
+  hostname: "localhost"
+  # address where lemmy should listen for incoming requests
+  bind: "0.0.0.0"
+  # port where lemmy should listen for incoming requests
+  port: 8536
+}

docker/dev/nginx.conf

@@ -1,51 +1,71 @@
-worker_processes  1;
+worker_processes 1;
 events {
-    worker_connections  1024;
+    worker_connections 1024;
 }
 http {
     upstream lemmy {
+        # this needs to map to the lemmy (server) docker service hostname
         server "lemmy:8536";
     }
     upstream lemmy-ui {
+        # this needs to map to the lemmy-ui docker service hostname
         server "lemmy-ui:1234";
     }
+
     server {
-      listen       1236;
+        # this is the port inside docker, not the public one yet
-      server_name  localhost;
+        listen 1236;
+        # change if needed, this is facing the public web
+        server_name localhost;
+        server_tokens off;
 
-      # frontend
+        gzip on;
-      location / {
+        gzip_types text/css application/javascript image/svg+xml;
-        set $proxpass "http://lemmy-ui";
+        gzip_vary on;
-        if ($http_accept = "application/activity+json") {
+
-          set $proxpass "http://lemmy";
+        # Upload limit, relevant for pictrs
+        client_max_body_size 20M;
+
+        add_header X-Frame-Options SAMEORIGIN;
+        add_header X-Content-Type-Options nosniff;
+        add_header X-XSS-Protection "1; mode=block";
+
+        # frontend general requests
+        location / {
+            # distinguish between ui requests and backend
+            # don't change lemmy-ui or lemmy here, they refer to the upstream definitions on top
+            set $proxpass "http://lemmy-ui";
+
+            if ($http_accept = "application/activity+json") {
+              set $proxpass "http://lemmy";
+            }
+            if ($http_accept = "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"") {
+              set $proxpass "http://lemmy";
+            }
+            if ($request_method = POST) {
+              set $proxpass "http://lemmy";
+            }
+            proxy_pass $proxpass;
+
+            rewrite ^(.+)/+$ $1 permanent;
+            # Send actual client IP upstream
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header Host $host;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         }
-        if ($http_accept = "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"") {
+
-          set $proxpass "http://lemmy";
+        # backend
+        location ~ ^/(api|pictrs|feeds|nodeinfo|.well-known) {
+            proxy_pass "http://lemmy";
+            # proxy common stuff
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+
+            # Send actual client IP upstream
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header Host $host;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         }
-        if ($request_method = POST) {
-          set $proxpass "http://lemmy";
-        }
-        proxy_pass $proxpass;
-
-        rewrite ^(.+)/+$ $1 permanent;
-
-        # Send actual client IP upstream
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header Host $host;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      }
-
-      # backend
-      location ~ ^/(api|pictrs|feeds|nodeinfo|.well-known) {
-        proxy_pass "http://lemmy";
-        proxy_http_version 1.1;
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection "upgrade";
-
-        # Add IP forwarding headers
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header Host $host;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      }
     }
 }

docker/dev/otel.yml

@@ -11,7 +11,8 @@ exporters:
   logging:
   jaeger:
     endpoint: jaeger:14250
-    insecure: true
+    tls:
+      insecure: true
 
 service:
   pipelines: