feat(metrics): add support for custom labels in Prometheus metrics (#979)
* feat: add dynamic labels support for Prometheus metrics - Add `toBoolPtr` function to convert a bool to a bool pointer - Add `contains` function to check if a key exists in a slice - Add `GetMetricLabels` method to `Config` struct to return unique metric labels from enabled endpoints - Change file permission notation from `0644` to `0o644` in `config_test.go` - Add `Labels` field to `Endpoint` struct for key-value pairs - Initialize Prometheus metrics with dynamic labels from configuration - Modify `PublishMetricsForEndpoint` to include dynamic labels - Add test for `GetMetricLabels` method in `config_test.go` - Update `watchdog` to pass labels to monitoring and execution functions Signed-off-by: appleboy <appleboy.tw@gmail.com> * refactor: refactor pointer conversion utility and update related tests - Rename `toBoolPtr` function to a generic `toPtr` function - Update tests to use the new `toPtr` function instead of `toBoolPtr` Signed-off-by: appleboy <appleboy.tw@gmail.com> * refactor: refactor utility functions and improve test coverage - Move `toPtr` and `contains` utility functions to a new file `util.go` Signed-off-by: appleboy <appleboy.tw@gmail.com> * missing labels parameter * refactor: reorder parameters in metrics-related functions and tests - Reorder parameters in `PublishMetricsForEndpoint` function - Update test cases to match the new parameter order in `PublishMetricsForEndpoint` - Reorder parameters in `monitor` function - Adjust `monitor` function calls to match the new parameter order - Reorder parameters in `execute` function call to `PublishMetricsForEndpoint` Signed-off-by: Bo-Yi Wu <appleboy.tw@gmail.com> * Update main.go * Update config/config.go * docs: improve documentation formatting, examples, and readability - Add multiple blank lines for spacing in the README file - Fix formatting issues in markdown tables - Correct deprecated formatting for Teams alerts - Replace single quotes with double quotes in JSON examples - Add new sections and examples for various configurations and endpoints - Improve readability and consistency in the documentation - Update links and references to examples and configurations Signed-off-by: Bo-Yi Wu <appleboy.tw@gmail.com> * docs: enhance custom labels support in Prometheus metrics - Add a section for custom labels in the README - Include an example configuration for custom labels in Prometheus metrics initialization Signed-off-by: Bo-Yi Wu <appleboy.tw@gmail.com> * refactor: rename and refactor metric labels to use ExtraLabels - Rename the endpoint metric labels field from Labels to ExtraLabels and update its YAML tag accordingly - Update code and tests to use ExtraLabels instead of Labels for metrics - Replace GetMetricLabels with GetUniqueExtraMetricLabels and adjust usages throughout the codebase - Ensure all metric publishing and monitoring functions accept and use the new extraLabels naming and semantics - Update tests to verify correct extraction and handling of ExtraLabels for enabled endpoints Signed-off-by: appleboy <appleboy.tw@gmail.com> * refactor: refactor parameter order for monitor and execute for consistency - Change the order of parameters for monitor and execute functions to group extraLabels consistently as the last argument before context. - Update all relevant function calls and signatures to reflect the new parameter order. - Replace usage of labels with extraLabels for clarity and consistency. Signed-off-by: appleboy <appleboy.tw@gmail.com> * test: improve initialization and labeling of Prometheus metrics - Add a test to verify that Prometheus metrics initialize correctly with extra labels. - Ensure metrics variables are properly initialized and not nil. - Check that WithLabelValues accepts both default and extra labels without causing a panic. Signed-off-by: appleboy <appleboy.tw@gmail.com> * test: improve Prometheus metrics testing for extra label handling - Remove a redundant test for WithLabelValues label length. - Add a new test to verify that extraLabels are correctly included in exported Prometheus metrics. Signed-off-by: appleboy <appleboy.tw@gmail.com> * refactor: refactor metrics to support custom Prometheus registries - Refactor metrics initialization to accept a custom Prometheus registry, defaulting to the global registry when nil - Replace promauto with direct metric construction and explicit registration - Update tests to use dedicated, isolated registries instead of the default global registry Signed-off-by: appleboy <appleboy.tw@gmail.com> * Revert README.md to a previous version * docs: document support for custom metric labels in endpoints - Add documentation section explaining support for custom labels on metrics - Provide YAML configuration example illustrating the new labels field for endpoints - Update table of contents to include the custom labels section Signed-off-by: appleboy <appleboy.tw@gmail.com> --------- Signed-off-by: appleboy <appleboy.tw@gmail.com> Signed-off-by: Bo-Yi Wu <appleboy.tw@gmail.com> Co-authored-by: TwiN <twin@linux.com>
This commit is contained in:
Bo-Yi Wu
@@ -3,16 +3,14 @@ package metrics
|
||||
import (
|
||||
"strconv"
|
||||
|
||||
"github.com/TwiN/gatus/v5/config"
|
||||
"github.com/TwiN/gatus/v5/config/endpoint"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
const namespace = "gatus" // The prefix of the metrics
|
||||
|
||||
var (
|
||||
initializedMetrics bool // Whether the metrics have been initialized
|
||||
|
||||
resultTotal *prometheus.CounterVec
|
||||
resultDurationSeconds *prometheus.GaugeVec
|
||||
resultConnectedTotal *prometheus.CounterVec
|
||||
@@ -21,64 +19,79 @@ var (
|
||||
resultEndpointSuccess *prometheus.GaugeVec
|
||||
)
|
||||
|
||||
func initializePrometheusMetrics() {
|
||||
resultTotal = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
func InitializePrometheusMetrics(cfg *config.Config, reg prometheus.Registerer) {
|
||||
if reg == nil {
|
||||
reg = prometheus.DefaultRegisterer
|
||||
}
|
||||
extraLabels := cfg.GetUniqueExtraMetricLabels()
|
||||
resultTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "results_total",
|
||||
Help: "Number of results per endpoint",
|
||||
}, []string{"key", "group", "name", "type", "success"})
|
||||
resultDurationSeconds = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
}, append([]string{"key", "group", "name", "type", "success"}, extraLabels...))
|
||||
reg.MustRegister(resultTotal)
|
||||
resultDurationSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Name: "results_duration_seconds",
|
||||
Help: "Duration of the request in seconds",
|
||||
}, []string{"key", "group", "name", "type"})
|
||||
resultConnectedTotal = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
}, append([]string{"key", "group", "name", "type"}, extraLabels...))
|
||||
reg.MustRegister(resultDurationSeconds)
|
||||
resultConnectedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "results_connected_total",
|
||||
Help: "Total number of results in which a connection was successfully established",
|
||||
}, []string{"key", "group", "name", "type"})
|
||||
resultCodeTotal = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
}, append([]string{"key", "group", "name", "type"}, extraLabels...))
|
||||
reg.MustRegister(resultConnectedTotal)
|
||||
resultCodeTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "results_code_total",
|
||||
Help: "Total number of results by code",
|
||||
}, []string{"key", "group", "name", "type", "code"})
|
||||
resultCertificateExpirationSeconds = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
}, append([]string{"key", "group", "name", "type", "code"}, extraLabels...))
|
||||
reg.MustRegister(resultCodeTotal)
|
||||
resultCertificateExpirationSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Name: "results_certificate_expiration_seconds",
|
||||
Help: "Number of seconds until the certificate expires",
|
||||
}, []string{"key", "group", "name", "type"})
|
||||
resultEndpointSuccess = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
}, append([]string{"key", "group", "name", "type"}, extraLabels...))
|
||||
reg.MustRegister(resultCertificateExpirationSeconds)
|
||||
resultEndpointSuccess = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Name: "results_endpoint_success",
|
||||
Help: "Displays whether or not the endpoint was a success",
|
||||
}, []string{"key", "group", "name", "type"})
|
||||
}, append([]string{"key", "group", "name", "type"}, extraLabels...))
|
||||
reg.MustRegister(resultEndpointSuccess)
|
||||
}
|
||||
|
||||
// PublishMetricsForEndpoint publishes metrics for the given endpoint and its result.
|
||||
// These metrics will be exposed at /metrics if the metrics are enabled
|
||||
func PublishMetricsForEndpoint(ep *endpoint.Endpoint, result *endpoint.Result) {
|
||||
if !initializedMetrics {
|
||||
initializePrometheusMetrics()
|
||||
initializedMetrics = true
|
||||
func PublishMetricsForEndpoint(ep *endpoint.Endpoint, result *endpoint.Result, extraLabels []string) {
|
||||
labelValues := []string{}
|
||||
for _, label := range extraLabels {
|
||||
if value, ok := ep.ExtraLabels[label]; ok {
|
||||
labelValues = append(labelValues, value)
|
||||
} else {
|
||||
labelValues = append(labelValues, "")
|
||||
}
|
||||
}
|
||||
|
||||
endpointType := ep.Type()
|
||||
resultTotal.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType), strconv.FormatBool(result.Success)).Inc()
|
||||
resultDurationSeconds.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType)).Set(result.Duration.Seconds())
|
||||
resultTotal.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType), strconv.FormatBool(result.Success)}, labelValues...)...).Inc()
|
||||
resultDurationSeconds.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType)}, labelValues...)...).Set(result.Duration.Seconds())
|
||||
if result.Connected {
|
||||
resultConnectedTotal.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType)).Inc()
|
||||
resultConnectedTotal.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType)}, labelValues...)...).Inc()
|
||||
}
|
||||
if result.DNSRCode != "" {
|
||||
resultCodeTotal.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType), result.DNSRCode).Inc()
|
||||
resultCodeTotal.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType), result.DNSRCode}, labelValues...)...).Inc()
|
||||
}
|
||||
if result.HTTPStatus != 0 {
|
||||
resultCodeTotal.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType), strconv.Itoa(result.HTTPStatus)).Inc()
|
||||
resultCodeTotal.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType), strconv.Itoa(result.HTTPStatus)}, labelValues...)...).Inc()
|
||||
}
|
||||
if result.CertificateExpiration != 0 {
|
||||
resultCertificateExpirationSeconds.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType)).Set(result.CertificateExpiration.Seconds())
|
||||
resultCertificateExpirationSeconds.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType)}, labelValues...)...).Set(result.CertificateExpiration.Seconds())
|
||||
}
|
||||
if result.Success {
|
||||
resultEndpointSuccess.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType)).Set(1)
|
||||
resultEndpointSuccess.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType)}, labelValues...)...).Set(1)
|
||||
} else {
|
||||
resultEndpointSuccess.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType)).Set(0)
|
||||
resultEndpointSuccess.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType)}, labelValues...)...).Set(0)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,13 +5,110 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/TwiN/gatus/v5/config"
|
||||
"github.com/TwiN/gatus/v5/config/endpoint"
|
||||
"github.com/TwiN/gatus/v5/config/endpoint/dns"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
)
|
||||
|
||||
// TestInitializePrometheusMetrics tests metrics initialization with extraLabels.
|
||||
// Note: Because of the global Prometheus registry, this test can only safely verify one label set per process.
|
||||
// If the function is called with a different set of labels for the same metric, a panic will occur.
|
||||
func TestInitializePrometheusMetrics(t *testing.T) {
|
||||
cfgWithExtras := &config.Config{
|
||||
Endpoints: []*endpoint.Endpoint{
|
||||
{
|
||||
Name: "TestEP",
|
||||
Group: "G",
|
||||
URL: "http://x/",
|
||||
ExtraLabels: map[string]string{
|
||||
"foo": "foo-val",
|
||||
"hello": "world-val",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
reg := prometheus.NewRegistry()
|
||||
InitializePrometheusMetrics(cfgWithExtras, reg)
|
||||
// Metrics variables should be non-nil
|
||||
if resultTotal == nil {
|
||||
t.Error("resultTotal metric not initialized")
|
||||
}
|
||||
if resultDurationSeconds == nil {
|
||||
t.Error("resultDurationSeconds metric not initialized")
|
||||
}
|
||||
if resultConnectedTotal == nil {
|
||||
t.Error("resultConnectedTotal metric not initialized")
|
||||
}
|
||||
if resultCodeTotal == nil {
|
||||
t.Error("resultCodeTotal metric not initialized")
|
||||
}
|
||||
if resultCertificateExpirationSeconds == nil {
|
||||
t.Error("resultCertificateExpirationSeconds metric not initialized")
|
||||
}
|
||||
if resultEndpointSuccess == nil {
|
||||
t.Error("resultEndpointSuccess metric not initialized")
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
t.Errorf("resultTotal.WithLabelValues panicked: %v", r)
|
||||
}
|
||||
}()
|
||||
_ = resultTotal.WithLabelValues("k", "g", "n", "ty", "true", "fval", "hval")
|
||||
}
|
||||
|
||||
// TestPublishMetricsForEndpoint_withExtraLabels ensures extraLabels are included in the exported metrics.
|
||||
func TestPublishMetricsForEndpoint_withExtraLabels(t *testing.T) {
|
||||
// Only test one label set per process due to Prometheus registry limits.
|
||||
reg := prometheus.NewRegistry()
|
||||
InitializePrometheusMetrics(&config.Config{
|
||||
Endpoints: []*endpoint.Endpoint{
|
||||
{
|
||||
Name: "ep-extra",
|
||||
URL: "https://sample.com",
|
||||
ExtraLabels: map[string]string{
|
||||
"foo": "my-foo",
|
||||
"bar": "my-bar",
|
||||
},
|
||||
},
|
||||
},
|
||||
}, reg)
|
||||
|
||||
ep := &endpoint.Endpoint{
|
||||
Name: "ep-extra",
|
||||
Group: "g1",
|
||||
URL: "https://sample.com",
|
||||
ExtraLabels: map[string]string{
|
||||
"foo": "my-foo",
|
||||
"bar": "my-bar",
|
||||
},
|
||||
}
|
||||
result := &endpoint.Result{
|
||||
HTTPStatus: 200,
|
||||
Connected: true,
|
||||
Duration: 2340 * time.Millisecond,
|
||||
Success: true,
|
||||
}
|
||||
// Order of extraLabels as per GetUniqueExtraMetricLabels is ["foo", "bar"]
|
||||
PublishMetricsForEndpoint(ep, result, []string{"foo", "bar"})
|
||||
|
||||
expected := `
|
||||
# HELP gatus_results_total Number of results per endpoint
|
||||
# TYPE gatus_results_total counter
|
||||
gatus_results_total{bar="my-bar",foo="my-foo",group="g1",key="g1_ep-extra",name="ep-extra",success="true",type="HTTP"} 1
|
||||
`
|
||||
err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expected), "gatus_results_total")
|
||||
if err != nil {
|
||||
t.Error("metrics export does not include extraLabels as expected:", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishMetricsForEndpoint(t *testing.T) {
|
||||
reg := prometheus.NewRegistry()
|
||||
InitializePrometheusMetrics(&config.Config{}, reg)
|
||||
|
||||
httpEndpoint := &endpoint.Endpoint{Name: "http-ep-name", Group: "http-ep-group", URL: "https://example.org"}
|
||||
PublishMetricsForEndpoint(httpEndpoint, &endpoint.Result{
|
||||
HTTPStatus: 200,
|
||||
@@ -23,8 +120,8 @@ func TestPublishMetricsForEndpoint(t *testing.T) {
|
||||
},
|
||||
Success: true,
|
||||
CertificateExpiration: 49 * time.Hour,
|
||||
})
|
||||
err := testutil.GatherAndCompare(prometheus.Gatherers{prometheus.DefaultGatherer}, bytes.NewBufferString(`
|
||||
}, []string{})
|
||||
err := testutil.GatherAndCompare(reg, bytes.NewBufferString(`
|
||||
# HELP gatus_results_code_total Total number of results by code
|
||||
# TYPE gatus_results_code_total counter
|
||||
gatus_results_code_total{code="200",group="http-ep-group",key="http-ep-group_http-ep-name",name="http-ep-name",type="HTTP"} 1
|
||||
@@ -57,8 +154,8 @@ gatus_results_endpoint_success{group="http-ep-group",key="http-ep-group_http-ep-
|
||||
},
|
||||
Success: false,
|
||||
CertificateExpiration: 47 * time.Hour,
|
||||
})
|
||||
err = testutil.GatherAndCompare(prometheus.Gatherers{prometheus.DefaultGatherer}, bytes.NewBufferString(`
|
||||
}, []string{})
|
||||
err = testutil.GatherAndCompare(reg, bytes.NewBufferString(`
|
||||
# HELP gatus_results_code_total Total number of results by code
|
||||
# TYPE gatus_results_code_total counter
|
||||
gatus_results_code_total{code="200",group="http-ep-group",key="http-ep-group_http-ep-name",name="http-ep-name",type="HTTP"} 2
|
||||
@@ -82,10 +179,12 @@ gatus_results_endpoint_success{group="http-ep-group",key="http-ep-group_http-ep-
|
||||
if err != nil {
|
||||
t.Errorf("Expected no errors but got: %v", err)
|
||||
}
|
||||
dnsEndpoint := &endpoint.Endpoint{Name: "dns-ep-name", Group: "dns-ep-group", URL: "8.8.8.8", DNSConfig: &dns.Config{
|
||||
QueryType: "A",
|
||||
QueryName: "example.com.",
|
||||
}}
|
||||
dnsEndpoint := &endpoint.Endpoint{
|
||||
Name: "dns-ep-name", Group: "dns-ep-group", URL: "8.8.8.8", DNSConfig: &dns.Config{
|
||||
QueryType: "A",
|
||||
QueryName: "example.com.",
|
||||
},
|
||||
}
|
||||
PublishMetricsForEndpoint(dnsEndpoint, &endpoint.Result{
|
||||
DNSRCode: "NOERROR",
|
||||
Connected: true,
|
||||
@@ -94,8 +193,8 @@ gatus_results_endpoint_success{group="http-ep-group",key="http-ep-group_http-ep-
|
||||
{Condition: "[DNS_RCODE] == NOERROR", Success: true},
|
||||
},
|
||||
Success: true,
|
||||
})
|
||||
err = testutil.GatherAndCompare(prometheus.Gatherers{prometheus.DefaultGatherer}, bytes.NewBufferString(`
|
||||
}, []string{})
|
||||
err = testutil.GatherAndCompare(reg, bytes.NewBufferString(`
|
||||
# HELP gatus_results_code_total Total number of results by code
|
||||
# TYPE gatus_results_code_total counter
|
||||
gatus_results_code_total{code="200",group="http-ep-group",key="http-ep-group_http-ep-name",name="http-ep-name",type="HTTP"} 2
|
||||
|
||||
Reference in New Issue
Block a user