debugd: collect pod logs

Signed-off-by: Paul Meyer <49727155+katexochen@users.noreply.github.com>

debugd/internal/debugd/logcollector/filebeat/Dockerfile

@@ -4,4 +4,6 @@ RUN dnf install -y https://artifacts.elastic.co/downloads/beats/filebeat/filebea
 
 COPY debugd/internal/debugd/logcollector/filebeat/filebeat.yml /usr/share/filebeat/filebeat.yml
 
+COPY debugd/internal/debugd/logcollector/filebeat/inputs.yml /usr/share/filebeat/inputs.d/inputs.yml
+
 ENTRYPOINT ["/usr/share/filebeat/bin/filebeat", "-e", "--path.home", "/usr/share/filebeat", "--path.data", "/usr/share/filebeat/data"]

debugd/internal/debugd/logcollector/filebeat/filebeat.yml

@@ -1,8 +1,3 @@
-filebeat.inputs:
-  - type: journald
-    enabled: true
-    id: everything
-
 output.logstash:
   hosts: ["localhost:5044"]
 
@@ -13,3 +8,20 @@ logging:
   to_files: false
   metrics.enabled: false
   level: warning
+
+filebeat.config:
+  inputs:
+    enabled: true
+    path: /usr/share/filebeat/inputs.d/*.yml
+    # reload.enabled: true
+    # reload.period: 10s
+
+processors:
+  - dissect:
+      when:
+        regexp:
+          log.file.path: "^/var/log/pods/*"
+      tokenizer: "/var/log/pods/%{namespace}_%{pod_name}_%{uid}/%{container_name}/%{run_id}.log"
+      field: "log.file.path"
+      target_prefix: "kubernetes"
+      ignore_failure: true

debugd/internal/debugd/logcollector/filebeat/inputs.yml

@@ -0,0 +1,9 @@
+- type: journald
+  enabled: true
+  id: journald
+
+- type: filestream
+  enabled: true
+  id: container
+  paths:
+    - /var/log/pods/*/*/*.log

debugd/internal/debugd/logcollector/logcollector.go

@@ -194,6 +194,7 @@ func startPod(ctx context.Context, logger *logger.Logger) error {
 		"--volume=/etc/machine-id:/etc/machine-id:ro",
 		"--volume=/run/systemd:/run/systemd:ro",
 		"--volume=/run/systemd/journal/socket:/run/systemd/journal/socket:rw",
+		"--volume=/run/state/var/log:/var/log:ro",
 		versions.FilebeatImage,
 	}
 	runFilebeatCmd := exec.CommandContext(ctx, "podman", runFilebeatArgs...)

debugd/internal/debugd/logcollector/logstash/templates/pipeline.conf

@@ -11,9 +11,11 @@ filter {
         remove_field => [
             "[agent]",
             "[journald]",
-            "[log]",
             "[syslog]",
-            "[systemd][invocation_id]"
+            "[systemd][invocation_id]",
+            "[event][original]",
+            "[log][offset]",
+            "[log][syslog]"
         ]
 
         # Tag with the provided metadata.