From 9a282df84686f2fa6de011ce76595de4af34afe5 Mon Sep 17 00:00:00 2001 From: Moritz Sanft <58110325+msanft@users.noreply.github.com> Date: Tue, 31 Oct 2023 12:09:27 +0100 Subject: [PATCH] ci: separate logs and metrics indices (#2544) * separate logs and metrics indices * tidy --- .../debugd/logcollector/logcollector.go | 2 -- debugd/logstash/templates/pipeline.conf | 30 ++++++++++++++----- hack/logcollector/cmd/template.go | 9 ------ hack/logcollector/internal/logstash.go | 30 ++++++++----------- 4 files changed, 36 insertions(+), 35 deletions(-) diff --git a/debugd/internal/debugd/logcollector/logcollector.go b/debugd/internal/debugd/logcollector/logcollector.go index 28b04a521..9e5c60165 100644 --- a/debugd/internal/debugd/logcollector/logcollector.go +++ b/debugd/internal/debugd/logcollector/logcollector.go @@ -93,7 +93,6 @@ func NewStartTrigger(ctx context.Context, wg *sync.WaitGroup, provider cloudprov pipelineConf := logstashConfInput{ Port: 5044, Host: openSearchHost, - IndexPrefix: "systemd-logs", InfoMap: infoMapM, Credentials: creds, } @@ -272,7 +271,6 @@ func startPod(ctx context.Context, logger *logger.Logger) error { type logstashConfInput struct { Port int Host string - IndexPrefix string InfoMap map[string]string Credentials credentials } diff --git a/debugd/logstash/templates/pipeline.conf b/debugd/logstash/templates/pipeline.conf index b67b4f6ab..26bcffc1c 100644 --- a/debugd/logstash/templates/pipeline.conf +++ b/debugd/logstash/templates/pipeline.conf @@ -55,12 +55,28 @@ filter { } output { - opensearch { - hosts => "{{ .Host }}" - index => "{{ .IndexPrefix }}-%{+YYYY.MM.dd}" - user => "{{ .Credentials.Username }}" - password => "{{ .Credentials.Password }}" - ssl => true - ssl_certificate_verification => true + if ([@metadata][beat] == "filebeat") { + # Logs, which are output by filebeat, go to the logs-index. + opensearch { + hosts => "{{ .Host }}" + # YYYY doesn't handle rolling over the year, so we use xxxx instead. + # See https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/541#issuecomment-270923437. + index => "logs-%{+xxxx.ww}" + user => "{{ .Credentials.Username }}" + password => "{{ .Credentials.Password }}" + ssl => true + ssl_certificate_verification => true + } + } else { + opensearch { + hosts => "{{ .Host }}" + # YYYY doesn't handle rolling over the year, so we use xxxx instead. + # See https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/541#issuecomment-270923437. + index => "metrics-%{+xxxx.ww}" + user => "{{ .Credentials.Username }}" + password => "{{ .Credentials.Password }}" + ssl => true + ssl_certificate_verification => true + } } } diff --git a/hack/logcollector/cmd/template.go b/hack/logcollector/cmd/template.go index 951ffb08d..8776a52df 100644 --- a/hack/logcollector/cmd/template.go +++ b/hack/logcollector/cmd/template.go @@ -28,7 +28,6 @@ func newTemplateCmd() *cobra.Command { must(templateCmd.MarkFlagRequired("username")) templateCmd.Flags().String("password", "", "OpenSearch password (required)") must(templateCmd.MarkFlagRequired("password")) - templateCmd.Flags().String("index-prefix", "systemd-logs", "Prefix for logging index (e.g. systemd-logs)") templateCmd.Flags().Int("port", 5045, "Logstash port") templateCmd.Flags().StringToString("fields", nil, "Additional fields for the Logstash pipeline") @@ -49,7 +48,6 @@ func runTemplate(cmd *cobra.Command, _ []string) error { flags.extraFields, flags.username, flags.password, - flags.indexPrefix, flags.port, ) if err := logstashPreparer.Prepare(flags.dir); err != nil { @@ -89,11 +87,6 @@ func parseTemplateFlags(cmd *cobra.Command) (templateFlags, error) { return templateFlags{}, fmt.Errorf("parse password string: %w", err) } - indexPrefix, err := cmd.Flags().GetString("index-prefix") - if err != nil { - return templateFlags{}, fmt.Errorf("parse index-prefix string: %w", err) - } - extraFields, err := cmd.Flags().GetStringToString("fields") if err != nil { return templateFlags{}, fmt.Errorf("parse fields map: %w", err) @@ -108,7 +101,6 @@ func parseTemplateFlags(cmd *cobra.Command) (templateFlags, error) { dir: dir, username: username, password: password, - indexPrefix: indexPrefix, extraFields: extraFields, port: port, }, nil @@ -118,7 +110,6 @@ type templateFlags struct { dir string username string password string - indexPrefix string extraFields fields.Fields port int } diff --git a/hack/logcollector/internal/logstash.go b/hack/logcollector/internal/logstash.go index 6ed7bb8ac..ea03365e6 100644 --- a/hack/logcollector/internal/logstash.go +++ b/hack/logcollector/internal/logstash.go @@ -29,33 +29,30 @@ const ( // LogstashPreparer prepares the Logstash Helm chart. type LogstashPreparer struct { - fh file.Handler - fields map[string]string - indexPrefix string - username string - password string - port int + fh file.Handler + fields map[string]string + username string + password string + port int templatePreparer } // NewLogstashPreparer returns a new LogstashPreparer. -func NewLogstashPreparer(fields map[string]string, username, password, indexPrefix string, port int) *LogstashPreparer { +func NewLogstashPreparer(fields map[string]string, username, password string, port int) *LogstashPreparer { return &LogstashPreparer{ - username: username, - password: password, - indexPrefix: indexPrefix, - fields: fields, - fh: file.NewHandler(afero.NewOsFs()), - port: port, + username: username, + password: password, + fields: fields, + fh: file.NewHandler(afero.NewOsFs()), + port: port, } } // Prepare prepares the Logstash Helm chart by templating the required files and placing them in the specified directory. func (p *LogstashPreparer) Prepare(dir string) error { templatedPipelineConf, err := p.template(logstashAssets, "templates/pipeline.conf", pipelineConfTemplate{ - InfoMap: p.fields, - Host: openSearchHost, - IndexPrefix: p.indexPrefix, + InfoMap: p.fields, + Host: openSearchHost, Credentials: Credentials{ Username: p.username, Password: p.password, @@ -134,7 +131,6 @@ type LogstashHelmValues struct { type pipelineConfTemplate struct { InfoMap map[string]string Host string - IndexPrefix string Credentials Credentials Port int }