Implement metrics collection for API server (#143)

api.go

@@ -8,13 +8,24 @@ import (
 	"strings"
 	"time"
 
+	"github.com/Luzifer/ots/pkg/metrics"
+	"github.com/Luzifer/ots/pkg/storage"
 	"github.com/gofrs/uuid"
 	"github.com/gorilla/mux"
 	"github.com/sirupsen/logrus"
 )
 
+const (
+	errorReasonInvalidJSON    = "invalid_json"
+	errorReasonSecretMissing  = "secret_missing"
+	errorReasonSecretSize     = "secret_size"
+	errorReasonStorageError   = "storage_error"
+	errorReasonSecretNotFound = "secret_not_found"
+)
+
 type apiServer struct {
-	store storage
+	collector *metrics.Collector
+	store     storage.Storage
 }
 
 type apiResponse struct {
@@ -29,8 +40,9 @@ type apiRequest struct {
 	Secret string `json:"secret"`
 }
 
-func newAPI(s storage) *apiServer {
+func newAPI(s storage.Storage, c *metrics.Collector) *apiServer {
 	return &apiServer{
+		collector: c,
 		store:     s,
 	}
 }
@@ -57,6 +69,7 @@ func (a apiServer) handleCreate(res http.ResponseWriter, r *http.Request) {
 	if strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
 		tmp := apiRequest{}
 		if err := json.NewDecoder(r.Body).Decode(&tmp); err != nil {
+			a.collector.CountSecretCreateError(errorReasonInvalidJSON)
 			a.errorResponse(res, http.StatusBadRequest, err, "decoding request body")
 			return
 		}
@@ -66,17 +79,20 @@ func (a apiServer) handleCreate(res http.ResponseWriter, r *http.Request) {
 	}
 
 	if secret == "" {
+		a.collector.CountSecretCreateError(errorReasonSecretMissing)
 		a.errorResponse(res, http.StatusBadRequest, errors.New("secret missing"), "")
 		return
 	}
 
 	if cust.MaxSecretSize > 0 && len(secret) > int(cust.MaxSecretSize) {
+		a.collector.CountSecretCreateError(errorReasonSecretSize)
 		a.errorResponse(res, http.StatusBadRequest, errors.New("secret size exceeds maximum"), "")
 		return
 	}
 
 	id, err := a.store.Create(secret, time.Duration(expiry)*time.Second)
 	if err != nil {
+		a.collector.CountSecretCreateError(errorReasonStorageError)
 		a.errorResponse(res, http.StatusInternalServerError, err, "creating secret")
 		return
 	}
@@ -86,6 +102,8 @@ func (a apiServer) handleCreate(res http.ResponseWriter, r *http.Request) {
 		expiresAt = func(v time.Time) *time.Time { return &v }(time.Now().UTC().Add(time.Duration(expiry) * time.Second))
 	}
 
+	a.collector.CountSecretCreated()
+	go updateStoredSecretsCount(a.store, a.collector)
 	a.jsonResponse(res, http.StatusCreated, apiResponse{
 		ExpiresAt: expiresAt,
 		Success:   true,
@@ -104,13 +122,18 @@ func (a apiServer) handleRead(res http.ResponseWriter, r *http.Request) {
 	secret, err := a.store.ReadAndDestroy(id)
 	if err != nil {
 		status := http.StatusInternalServerError
-		if err == errSecretNotFound {
+		if errors.Is(err, storage.ErrSecretNotFound) {
+			a.collector.CountSecretReadError(errorReasonSecretNotFound)
 			status = http.StatusNotFound
+		} else {
+			a.collector.CountSecretReadError(errorReasonStorageError)
 		}
 		a.errorResponse(res, status, err, "reading & destroying secret")
 		return
 	}
 
+	a.collector.CountSecretRead()
+	go updateStoredSecretsCount(a.store, a.collector)
 	a.jsonResponse(res, http.StatusOK, apiResponse{
 		Success: true,
 		Secret:  secret,

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/gofrs/uuid v4.4.0+incompatible
 	github.com/gorilla/mux v1.8.0
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.17.0
 	github.com/redis/go-redis/v9 v9.2.1
 	github.com/sirupsen/logrus v1.9.3
 )
@@ -21,18 +22,25 @@ require (
 require (
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/uuid v1.3.1 // indirect
 	github.com/huandu/xstrings v1.4.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.11.1 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/spf13/cast v1.5.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/crypto v0.14.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/validator.v2 v2.0.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go.sum

@@ -9,6 +9,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0
 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
 github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
 github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
@@ -26,6 +28,11 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
 github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -43,6 +50,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
+github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
 github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
@@ -59,10 +68,18 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
+github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
+github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
+github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
+github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
+github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
+github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
+github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
 github.com/redis/go-redis/v9 v9.2.1 h1:WlYJg71ODF0dVspZZCpYmoF1+U1Jjk9Rwd7pq6QmlCg=
 github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
-github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
-github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
@@ -92,6 +109,7 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU=
 golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -116,6 +134,11 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=

helpers.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"net"
+	"net/http"
+
+	"github.com/Luzifer/ots/pkg/metrics"
+	"github.com/Luzifer/ots/pkg/storage"
+	"github.com/sirupsen/logrus"
+)
+
+func requestInSubnetList(r *http.Request, subnets []string) bool {
+	if len(subnets) == 0 {
+		// No subnets specififed: None allowed (without doing the parsing)
+		return false
+	}
+
+	remote, _, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		logrus.WithError(err).Error("parsing remote address")
+		return false
+	}
+
+	remoteIP := net.ParseIP(remote)
+	if remoteIP == nil {
+		logrus.WithError(err).Error("parsing remote address")
+		return false
+	}
+
+	for _, sn := range subnets {
+		_, netw, err := net.ParseCIDR(sn)
+		if err != nil {
+			logrus.WithError(err).WithField("subnet", sn).Warn("invalid subnet specified")
+			continue
+		}
+
+		if netw.Contains(remoteIP) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func updateStoredSecretsCount(store storage.Storage, collector *metrics.Collector) {
+	n, err := store.Count()
+	if err != nil {
+		logrus.WithError(err).Error("counting stored secrets")
+		return
+	}
+	collector.UpdateSecretsCount(n)
+}

main.go

@@ -19,6 +19,7 @@ import (
 	file_helpers "github.com/Luzifer/go_helpers/v2/file"
 	http_helpers "github.com/Luzifer/go_helpers/v2/http"
 	"github.com/Luzifer/ots/pkg/customization"
+	"github.com/Luzifer/ots/pkg/metrics"
 	"github.com/Luzifer/rconfig/v2"
 )
 
@@ -99,6 +100,9 @@ func main() {
 		os.Exit(0)
 	}
 
+	// Initialize metrics collector
+	collector := metrics.New()
+
 	// Initialize index template in order not to parse it multiple times
 	source, err := assets.ReadFile("index.html")
 	if err != nil {
@@ -111,27 +115,49 @@ func main() {
 	if err != nil {
 		logrus.WithError(err).Fatal("initializing storage")
 	}
-	api := newAPI(store)
+	api := newAPI(store, collector)
 
+	// Initialize server
 	r := mux.NewRouter()
-	r.Use(http_helpers.GzipHandler)
 
 	api.Register(r.PathPrefix("/api").Subrouter())
 
-	r.HandleFunc("/", handleIndex)
+	r.Handle("/metrics", metrics.Handler()).
-	r.PathPrefix("/").HandlerFunc(assetDelivery)
+		Methods(http.MethodGet).
+		MatcherFunc(func(r *http.Request, _ *mux.RouteMatch) bool {
+			return requestInSubnetList(r, cust.MetricsAllowedSubnets)
+		})
 
+	r.HandleFunc("/", handleIndex).
+		Methods(http.MethodGet)
+	r.PathPrefix("/").HandlerFunc(assetDelivery).
+		Methods(http.MethodGet)
+
+	var hdl http.Handler = r
+	hdl = http_helpers.GzipHandler(hdl)
+	hdl = http_helpers.NewHTTPLogHandlerWithLogger(hdl, logrus.StandardLogger())
+
+	server := &http.Server{
+		Addr:              cfg.Listen,
+		Handler:           hdl,
+		ReadHeaderTimeout: time.Second,
+	}
+
+	// Start periodic stored metrics update (required for multi-instance
+	// OTS hosting as other instances will create / delete secrets and
+	// we need to keep up with that)
+	go func() {
+		for t := time.NewTicker(time.Minute); ; <-t.C {
+			updateStoredSecretsCount(store, collector)
+		}
+	}()
+
+	// Start server
 	logrus.WithFields(logrus.Fields{
 		"secret_expiry": time.Duration(cfg.SecretExpiry) * time.Second,
 		"version":       version,
 	}).Info("ots started")
 
-	server := &http.Server{
-		Addr:              cfg.Listen,
-		Handler:           http_helpers.NewHTTPLogHandlerWithLogger(r, logrus.StandardLogger()),
-		ReadHeaderTimeout: time.Second,
-	}
-
 	if err = server.ListenAndServe(); err != nil {
 		logrus.WithError(err).Fatal("HTTP server quit unexpectedly")
 	}

pkg/customization/customize.go

@@ -30,6 +30,7 @@ type (
 		MaxAttachmentSizeTotal int64  `json:"maxAttachmentSizeTotal" yaml:"maxAttachmentSizeTotal"`
 
 		MaxSecretSize         int64    `json:"-" yaml:"maxSecretSize"`
+		MetricsAllowedSubnets []string `json:"-" yaml:"metricsAllowedSubnets"`
 		OverlayFSPath         string   `json:"-" yaml:"overlayFSPath"`
 		UseFormalLanguage     bool     `json:"-" yaml:"useFormalLanguage"`
 	}

pkg/metrics/metrics.go

@@ -0,0 +1,99 @@
+// Package metrics provides an abstraction around metrics collection
+// in order to bundle all metrics related calls in one location
+package metrics
+
+import (
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+const (
+	metricSecretsCreated      = "secrets_created"
+	metricSecretsRead         = "secrets_read"
+	metricSecretsCreateErrors = "secrets_create_errors"
+	meticsSecretsReadErrors   = "secrets_read_errors"
+	metricsSecretsStored      = "secrets_stored"
+
+	labelReason = "reason"
+
+	namespace = "ots"
+)
+
+type (
+	// Collector contains all required methods to collect metrics
+	// and to populate them into the Handler
+	Collector struct {
+		secretsCreated      prometheus.Counter
+		secretsRead         prometheus.Counter
+		secretsCreateErrors *prometheus.CounterVec
+		secretsReadErrors   *prometheus.CounterVec
+		secretsStored       prometheus.Gauge
+	}
+)
+
+// Handler returns the handler to be registered at /metrics
+func Handler() http.Handler { return promhttp.Handler() }
+
+// New creates a new Collector and registers the metrics
+func New() *Collector {
+	return &Collector{
+		secretsCreated: promauto.NewCounter(prometheus.CounterOpts{
+			Namespace: namespace,
+			Name:      metricSecretsCreated,
+			Help:      "number of successfully created secrets",
+		}),
+
+		secretsRead: promauto.NewCounter(prometheus.CounterOpts{
+			Namespace: namespace,
+			Name:      metricSecretsRead,
+			Help:      "number of fetched (and destroyed) secrets",
+		}),
+
+		secretsCreateErrors: promauto.NewCounterVec(prometheus.CounterOpts{
+			Namespace: namespace,
+			Name:      metricSecretsCreateErrors,
+			Help:      "number of errors on secret creation for each reason",
+		}, []string{labelReason}),
+
+		secretsReadErrors: promauto.NewCounterVec(prometheus.CounterOpts{
+			Namespace: namespace,
+			Name:      meticsSecretsReadErrors,
+			Help:      "number of read-errors for each reason",
+		}, []string{labelReason}),
+
+		secretsStored: promauto.NewGauge(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      metricsSecretsStored,
+			Help:      "number of secrets currently held in the backend store",
+		}),
+	}
+}
+
+// CountSecretCreated signalizes a secret has successfully been created
+func (c Collector) CountSecretCreated() { c.secretsCreated.Inc() }
+
+// CountSecretRead signalizes a secret has successfully been read and destroyed
+func (c Collector) CountSecretRead() { c.secretsRead.Inc() }
+
+// CountSecretCreateError signalizes an error occurred during secret
+// creation. The reason must not be the error.Error() but a simple
+// static string describing the error.
+func (c Collector) CountSecretCreateError(reason string) {
+	c.secretsCreateErrors.WithLabelValues(reason).Inc()
+}
+
+// CountSecretReadError signalizes an error occurred during secret
+// read. The reason must not be the error.Error() but a simple
+// static string describing the error.
+func (c Collector) CountSecretReadError(reason string) {
+	c.secretsReadErrors.WithLabelValues(reason).Inc()
+}
+
+// UpdateSecretsCount sets the current amount of secrets stored in the
+// backend storage
+func (c Collector) UpdateSecretsCount(count int64) {
+	c.secretsStored.Set(float64(count))
+}

pkg/storage/memory/memory.go

@@ -0,0 +1,76 @@
+// Package memory implements a pure in-memory store for secrets which
+// is suitable for testing and should not be used for productive use
+package memory
+
+import (
+	"sync"
+	"time"
+
+	"github.com/Luzifer/ots/pkg/storage"
+	"github.com/gofrs/uuid"
+)
+
+type (
+	memStorageSecret struct {
+		Expiry time.Time
+		Secret string
+	}
+
+	storageMem struct {
+		sync.RWMutex
+		store map[string]memStorageSecret
+	}
+)
+
+// New creates a new In-Mem storage
+func New() storage.Storage {
+	return &storageMem{
+		store: make(map[string]memStorageSecret),
+	}
+}
+
+func (s *storageMem) Count() (int64, error) {
+	s.RLock()
+	defer s.RUnlock()
+
+	return int64(len(s.store)), nil
+}
+
+func (s *storageMem) Create(secret string, expireIn time.Duration) (string, error) {
+	s.Lock()
+	defer s.Unlock()
+
+	var (
+		expire time.Time
+		id     = uuid.Must(uuid.NewV4()).String()
+	)
+
+	if expireIn > 0 {
+		expire = time.Now().Add(expireIn)
+	}
+
+	s.store[id] = memStorageSecret{
+		Expiry: expire,
+		Secret: secret,
+	}
+
+	return id, nil
+}
+
+func (s *storageMem) ReadAndDestroy(id string) (string, error) {
+	s.Lock()
+	defer s.Unlock()
+
+	secret, ok := s.store[id]
+	if !ok {
+		return "", storage.ErrSecretNotFound
+	}
+
+	defer delete(s.store, id)
+
+	if !secret.Expiry.IsZero() && secret.Expiry.Before(time.Now()) {
+		return "", storage.ErrSecretNotFound
+	}
+
+	return secret.Secret, nil
+}

pkg/storage/redis/redis.go

@@ -1,24 +1,30 @@
-package main
+// Package redis implements a Redis backed storage for secrets
+package redis
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"strings"
 	"time"
 
+	"github.com/Luzifer/ots/pkg/storage"
 	"github.com/gofrs/uuid"
-	"github.com/pkg/errors"
 	redis "github.com/redis/go-redis/v9"
 )
 
-const redisDefaultPrefix = "io.luzifer.ots"
+const (
+	redisDefaultPrefix = "io.luzifer.ots"
+	redisScanCount     = 10
+)
 
 type storageRedis struct {
 	conn *redis.Client
 }
 
-func newStorageRedis() (storage, error) {
+// New returns a new Redis backed storage
+func New() (storage.Storage, error) {
 	if os.Getenv("REDIS_URL") == "" {
 		return nil, fmt.Errorf("REDIS_URL environment variable not set")
 	}
@@ -30,7 +36,7 @@ func newStorageRedis() (storage, error) {
 	// in order to maintain backwards compatibility
 	opt, err := redis.ParseURL(strings.Replace(os.Getenv("REDIS_URL"), "tcp://", "redis://", 1))
 	if err != nil {
-		return nil, errors.Wrap(err, "parsing REDIS_URL")
+		return nil, fmt.Errorf("parsing REDIS_URL: %w", err)
 	}
 
 	s := &storageRedis{
@@ -40,24 +46,50 @@ func newStorageRedis() (storage, error) {
 	return s, nil
 }
 
+func (s storageRedis) Count() (n int64, err error) {
+	var cursor uint64
+
+	for {
+		var keys []string
+
+		keys, cursor, err = s.conn.Scan(context.Background(), cursor, s.redisKey("*"), redisScanCount).Result()
+		if err != nil {
+			return n, fmt.Errorf("scanning stored keys: %w", err)
+		}
+
+		n += int64(len(keys))
+		if cursor == 0 {
+			break
+		}
+	}
+
+	return n, nil
+}
+
 func (s storageRedis) Create(secret string, expireIn time.Duration) (string, error) {
 	id := uuid.Must(uuid.NewV4()).String()
 	err := s.conn.Set(context.Background(), s.redisKey(id), secret, expireIn).Err()
+	if err != nil {
+		return "", fmt.Errorf("writing redis key: %w", err)
+	}
 
-	return id, errors.Wrap(err, "writing redis key")
+	return id, nil
 }
 
 func (s storageRedis) ReadAndDestroy(id string) (string, error) {
 	secret, err := s.conn.Get(context.Background(), s.redisKey(id)).Result()
 	if err != nil {
 		if errors.Is(err, redis.Nil) {
-			return "", errSecretNotFound
+			return "", storage.ErrSecretNotFound
 		}
-		return "", errors.Wrap(err, "getting key")
+		return "", fmt.Errorf("getting key: %w", err)
 	}
 
 	err = s.conn.Del(context.Background(), s.redisKey(id)).Err()
-	return secret, errors.Wrap(err, "deleting key")
+	if err != nil {
+		return secret, fmt.Errorf("deleting key: %w", err)
+	}
+	return secret, nil
 }
 
 func (storageRedis) redisKey(id string) string {

pkg/storage/storage.go

@@ -0,0 +1,21 @@
+// Package storage describes the requirements a storage provider
+// has to fulfill ot be usable in OTS
+package storage
+
+import (
+	"errors"
+	"time"
+)
+
+type (
+	// Storage is the interface to implement in each storage provider
+	Storage interface {
+		Count() (int64, error)
+		Create(secret string, expireIn time.Duration) (string, error)
+		ReadAndDestroy(id string) (string, error)
+	}
+)
+
+// ErrSecretNotFound is a generic error to be returned when a secret
+// does not exist in the backend. It will then be handled by API.
+var ErrSecretNotFound = errors.New("secret not found")

storage.go

@@ -1,24 +1,25 @@
 package main
 
 import (
-	"errors"
 	"fmt"
-	"time"
+
+	"github.com/Luzifer/ots/pkg/storage"
+	"github.com/Luzifer/ots/pkg/storage/memory"
+	"github.com/Luzifer/ots/pkg/storage/redis"
 )
 
-var errSecretNotFound = errors.New("secret not found")
+func getStorageByType(t string) (storage.Storage, error) {
-
-type storage interface {
-	Create(secret string, expireIn time.Duration) (string, error)
-	ReadAndDestroy(id string) (string, error)
-}
-
-func getStorageByType(t string) (storage, error) {
 	switch t {
 	case "mem":
-		return newStorageMem(), nil
+		return memory.New(), nil
+
 	case "redis":
-		return newStorageRedis()
+		s, err := redis.New()
+		if err != nil {
+			return s, fmt.Errorf("creating redis storage: %w", err)
+		}
+		return s, nil
+
 	default:
 		return nil, fmt.Errorf("storage type %q not found", t)
 	}

storage_mem.go

@@ -1,55 +0,0 @@
-package main
-
-import (
-	"time"
-
-	"github.com/gofrs/uuid"
-)
-
-type memStorageSecret struct {
-	Expiry time.Time
-	Secret string
-}
-
-type storageMem struct {
-	store map[string]memStorageSecret
-}
-
-func newStorageMem() storage {
-	return &storageMem{
-		store: make(map[string]memStorageSecret),
-	}
-}
-
-func (s storageMem) Create(secret string, expireIn time.Duration) (string, error) {
-	var (
-		expire time.Time
-		id     = uuid.Must(uuid.NewV4()).String()
-	)
-
-	if expireIn > 0 {
-		expire = time.Now().Add(expireIn)
-	}
-
-	s.store[id] = memStorageSecret{
-		Expiry: expire,
-		Secret: secret,
-	}
-
-	return id, nil
-}
-
-func (s storageMem) ReadAndDestroy(id string) (string, error) {
-	secret, ok := s.store[id]
-	if !ok {
-		return "", errSecretNotFound
-	}
-
-	defer delete(s.store, id)
-
-	if !secret.Expiry.IsZero() && secret.Expiry.Before(time.Now()) {
-		return "", errSecretNotFound
-	}
-
-	return secret.Secret, nil
-}