ci: separate logs and metrics indices (#2544)

* separate logs and metrics indices

* tidy

debugd/internal/debugd/logcollector/logcollector.go

@@ -93,7 +93,6 @@ func NewStartTrigger(ctx context.Context, wg *sync.WaitGroup, provider cloudprov
 			pipelineConf := logstashConfInput{
 				Port:        5044,
 				Host:        openSearchHost,
-				IndexPrefix: "systemd-logs",
 				InfoMap:     infoMapM,
 				Credentials: creds,
 			}
@@ -272,7 +271,6 @@ func startPod(ctx context.Context, logger *logger.Logger) error {
 type logstashConfInput struct {
 	Port        int
 	Host        string
-	IndexPrefix string
 	InfoMap     map[string]string
 	Credentials credentials
 }

debugd/logstash/templates/pipeline.conf

@@ -55,12 +55,28 @@ filter {
 }
 
 output {
-    opensearch {
-        hosts => "{{ .Host }}"
-        index => "{{ .IndexPrefix }}-%{+YYYY.MM.dd}"
-        user => "{{ .Credentials.Username }}"
-        password => "{{ .Credentials.Password }}"
-        ssl => true
-        ssl_certificate_verification => true
+    if ([@metadata][beat] == "filebeat") {
+        # Logs, which are output by filebeat, go to the logs-index.
+        opensearch {
+            hosts => "{{ .Host }}"
+            # YYYY doesn't handle rolling over the year, so we use xxxx instead.
+            # See https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/541#issuecomment-270923437.
+            index => "logs-%{+xxxx.ww}"
+            user => "{{ .Credentials.Username }}"
+            password => "{{ .Credentials.Password }}"
+            ssl => true
+            ssl_certificate_verification => true
+        }
+    } else {
+        opensearch {
+            hosts => "{{ .Host }}"
+            # YYYY doesn't handle rolling over the year, so we use xxxx instead.
+            # See https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/541#issuecomment-270923437.
+            index => "metrics-%{+xxxx.ww}"
+            user => "{{ .Credentials.Username }}"
+            password => "{{ .Credentials.Password }}"
+            ssl => true
+            ssl_certificate_verification => true
+        }
     }
 }

hack/logcollector/cmd/template.go

@@ -28,7 +28,6 @@ func newTemplateCmd() *cobra.Command {
 	must(templateCmd.MarkFlagRequired("username"))
 	templateCmd.Flags().String("password", "", "OpenSearch password (required)")
 	must(templateCmd.MarkFlagRequired("password"))
-	templateCmd.Flags().String("index-prefix", "systemd-logs", "Prefix for logging index (e.g. systemd-logs)")
 	templateCmd.Flags().Int("port", 5045, "Logstash port")
 	templateCmd.Flags().StringToString("fields", nil, "Additional fields for the Logstash pipeline")
 
@@ -49,7 +48,6 @@ func runTemplate(cmd *cobra.Command, _ []string) error {
 		flags.extraFields,
 		flags.username,
 		flags.password,
-		flags.indexPrefix,
 		flags.port,
 	)
 	if err := logstashPreparer.Prepare(flags.dir); err != nil {
@@ -89,11 +87,6 @@ func parseTemplateFlags(cmd *cobra.Command) (templateFlags, error) {
 		return templateFlags{}, fmt.Errorf("parse password string: %w", err)
 	}
 
-	indexPrefix, err := cmd.Flags().GetString("index-prefix")
-	if err != nil {
-		return templateFlags{}, fmt.Errorf("parse index-prefix string: %w", err)
-	}
-
 	extraFields, err := cmd.Flags().GetStringToString("fields")
 	if err != nil {
 		return templateFlags{}, fmt.Errorf("parse fields map: %w", err)
@@ -108,7 +101,6 @@ func parseTemplateFlags(cmd *cobra.Command) (templateFlags, error) {
 		dir:         dir,
 		username:    username,
 		password:    password,
-		indexPrefix: indexPrefix,
 		extraFields: extraFields,
 		port:        port,
 	}, nil
@@ -118,7 +110,6 @@ type templateFlags struct {
 	dir         string
 	username    string
 	password    string
-	indexPrefix string
 	extraFields fields.Fields
 	port        int
 }

hack/logcollector/internal/logstash.go

@@ -29,33 +29,30 @@ const (
 
 // LogstashPreparer prepares the Logstash Helm chart.
 type LogstashPreparer struct {
-	fh          file.Handler
-	fields      map[string]string
-	indexPrefix string
-	username    string
-	password    string
-	port        int
+	fh       file.Handler
+	fields   map[string]string
+	username string
+	password string
+	port     int
 	templatePreparer
 }
 
 // NewLogstashPreparer returns a new LogstashPreparer.
-func NewLogstashPreparer(fields map[string]string, username, password, indexPrefix string, port int) *LogstashPreparer {
+func NewLogstashPreparer(fields map[string]string, username, password string, port int) *LogstashPreparer {
 	return &LogstashPreparer{
-		username:    username,
-		password:    password,
-		indexPrefix: indexPrefix,
-		fields:      fields,
-		fh:          file.NewHandler(afero.NewOsFs()),
-		port:        port,
+		username: username,
+		password: password,
+		fields:   fields,
+		fh:       file.NewHandler(afero.NewOsFs()),
+		port:     port,
 	}
 }
 
 // Prepare prepares the Logstash Helm chart by templating the required files and placing them in the specified directory.
 func (p *LogstashPreparer) Prepare(dir string) error {
 	templatedPipelineConf, err := p.template(logstashAssets, "templates/pipeline.conf", pipelineConfTemplate{
-		InfoMap:     p.fields,
-		Host:        openSearchHost,
-		IndexPrefix: p.indexPrefix,
+		InfoMap: p.fields,
+		Host:    openSearchHost,
 		Credentials: Credentials{
 			Username: p.username,
 			Password: p.password,
@@ -134,7 +131,6 @@ type LogstashHelmValues struct {
 type pipelineConfTemplate struct {
 	InfoMap     map[string]string
 	Host        string
-	IndexPrefix string
 	Credentials Credentials
 	Port        int
 }