constellation/hack/logcollector/internal/filebeat.go
Moritz Sanft f4b2d02194
ci: collect cluster metrics to OpenSearch (#2347)
* add Metricbeat deployment to debugd

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* set metricbeat debugd image version

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* fix k8s deployment

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* use 2 separate deployments

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* only deploy via k8s in non-debug-images

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* add missing tilde

* remove k8s metrics

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* unify flag

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* add cloud metadata processor to filebeat

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* ci: fix debugd logcollection (#2355)

* add missing keyvault access role

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* bump logstash image version

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* bump filebeat / metricbeat image version

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* log used image version

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* use debugging image versions

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* increase wait timeout for image upload

* add cloud metadata processor to filebeat

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* fix template locations in container

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* fix image version typo

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* add filebeat / metricbeat users

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* remove user additions

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* update workflow step name

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* only mount config files

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* document potential rc

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* fix IAM permissions in workflow

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* fix AWS permissions

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* tidy

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* add missing workflow input

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* rename action

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* pin image versions

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* remove unnecessary workflow inputs

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

---------

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* add refStream input

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* remove inputs.yml dep

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* increase system metric period

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* fix linkchecker

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

---------

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>
2023-09-27 16:17:31 +02:00

114 lines
3.1 KiB
Go

/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package internal
import (
"embed"
"fmt"
"path/filepath"
"github.com/edgelesssys/constellation/v2/debugd/filebeat"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/spf13/afero"
"gopkg.in/yaml.v3"
)
var (
//go:embed templates/filebeat/*
filebeatHelmAssets embed.FS
filebeatAssets = filebeat.Assets
)
// FilebeatPreparer prepares the Filebeat Helm chart.
type FilebeatPreparer struct {
fh file.Handler
port int
templatePreparer
}
// NewFilebeatPreparer returns a new FilebeatPreparer.
func NewFilebeatPreparer(port int) *FilebeatPreparer {
return &FilebeatPreparer{
fh: file.NewHandler(afero.NewOsFs()),
port: port,
}
}
// Prepare prepares the Filebeat Helm chart by templating the filebeat.yml and inputs.yml files and placing them in the specified directory.
func (p *FilebeatPreparer) Prepare(dir string) error {
templatedFilebeatYaml, err := p.template(filebeatAssets, "templates/filebeat.yml", FilebeatTemplateData{
LogstashHost: fmt.Sprintf("logstash-logstash:%d", p.port),
AddCloudMetadata: true,
})
if err != nil {
return fmt.Errorf("template filebeat.yml: %w", err)
}
rawHelmValues, err := filebeatHelmAssets.ReadFile("templates/filebeat/values.yml")
if err != nil {
return fmt.Errorf("read values.yml: %w", err)
}
helmValuesYaml := &FilebeatHelmValues{}
if err := yaml.Unmarshal(rawHelmValues, helmValuesYaml); err != nil {
return fmt.Errorf("unmarshal values.yml: %w", err)
}
helmValuesYaml.Daemonset.FilebeatConfig.FilebeatYml = templatedFilebeatYaml.String()
helmValues, err := yaml.Marshal(helmValuesYaml)
if err != nil {
return fmt.Errorf("marshal values.yml: %w", err)
}
if err = p.fh.Write(filepath.Join(dir, "filebeat", "values.yml"), helmValues, file.OptMkdirAll); err != nil {
return fmt.Errorf("write values.yml: %w", err)
}
return nil
}
// FilebeatTemplateData is template data.
type FilebeatTemplateData struct {
LogstashHost string
AddCloudMetadata bool
}
// FilebeatHelmValues repesents the Helm values.yml.
type FilebeatHelmValues struct {
Image string `yaml:"image"`
ImageTag string `yaml:"imageTag"`
Daemonset struct {
Enabled bool `yaml:"enabled"`
FilebeatConfig struct {
FilebeatYml string `yaml:"filebeat.yml"`
} `yaml:"filebeatConfig"`
ExtraEnvs []interface{} `yaml:"extraEnvs"`
SecretMounts []interface{} `yaml:"secretMounts"`
Tolerations []struct {
Key string `yaml:"key"`
Operator string `yaml:"operator"`
Effect string `yaml:"effect"`
} `yaml:"tolerations"`
SecurityContext struct {
Privileged bool `yaml:"privileged"`
RunAsUser int `yaml:"runAsUser"`
} `yaml:"securityContext"`
ExtraVolumeMounts []struct {
Name string `yaml:"name"`
MountPath string `yaml:"mountPath"`
ReadOnly bool `yaml:"readOnly"`
} `yaml:"extraVolumeMounts"`
ExtraVolumes []struct {
Name string `yaml:"name"`
HostPath struct {
Path string `yaml:"path"`
Type string `yaml:"type"`
} `yaml:"hostPath"`
} `yaml:"extraVolumes"`
} `yaml:"daemonset"`
}