feat(metrics): add support for custom labels in Prometheus metrics (#979)
* feat: add dynamic labels support for Prometheus metrics - Add `toBoolPtr` function to convert a bool to a bool pointer - Add `contains` function to check if a key exists in a slice - Add `GetMetricLabels` method to `Config` struct to return unique metric labels from enabled endpoints - Change file permission notation from `0644` to `0o644` in `config_test.go` - Add `Labels` field to `Endpoint` struct for key-value pairs - Initialize Prometheus metrics with dynamic labels from configuration - Modify `PublishMetricsForEndpoint` to include dynamic labels - Add test for `GetMetricLabels` method in `config_test.go` - Update `watchdog` to pass labels to monitoring and execution functions Signed-off-by: appleboy <appleboy.tw@gmail.com> * refactor: refactor pointer conversion utility and update related tests - Rename `toBoolPtr` function to a generic `toPtr` function - Update tests to use the new `toPtr` function instead of `toBoolPtr` Signed-off-by: appleboy <appleboy.tw@gmail.com> * refactor: refactor utility functions and improve test coverage - Move `toPtr` and `contains` utility functions to a new file `util.go` Signed-off-by: appleboy <appleboy.tw@gmail.com> * missing labels parameter * refactor: reorder parameters in metrics-related functions and tests - Reorder parameters in `PublishMetricsForEndpoint` function - Update test cases to match the new parameter order in `PublishMetricsForEndpoint` - Reorder parameters in `monitor` function - Adjust `monitor` function calls to match the new parameter order - Reorder parameters in `execute` function call to `PublishMetricsForEndpoint` Signed-off-by: Bo-Yi Wu <appleboy.tw@gmail.com> * Update main.go * Update config/config.go * docs: improve documentation formatting, examples, and readability - Add multiple blank lines for spacing in the README file - Fix formatting issues in markdown tables - Correct deprecated formatting for Teams alerts - Replace single quotes with double quotes in JSON examples - Add new sections and examples for various configurations and endpoints - Improve readability and consistency in the documentation - Update links and references to examples and configurations Signed-off-by: Bo-Yi Wu <appleboy.tw@gmail.com> * docs: enhance custom labels support in Prometheus metrics - Add a section for custom labels in the README - Include an example configuration for custom labels in Prometheus metrics initialization Signed-off-by: Bo-Yi Wu <appleboy.tw@gmail.com> * refactor: rename and refactor metric labels to use ExtraLabels - Rename the endpoint metric labels field from Labels to ExtraLabels and update its YAML tag accordingly - Update code and tests to use ExtraLabels instead of Labels for metrics - Replace GetMetricLabels with GetUniqueExtraMetricLabels and adjust usages throughout the codebase - Ensure all metric publishing and monitoring functions accept and use the new extraLabels naming and semantics - Update tests to verify correct extraction and handling of ExtraLabels for enabled endpoints Signed-off-by: appleboy <appleboy.tw@gmail.com> * refactor: refactor parameter order for monitor and execute for consistency - Change the order of parameters for monitor and execute functions to group extraLabels consistently as the last argument before context. - Update all relevant function calls and signatures to reflect the new parameter order. - Replace usage of labels with extraLabels for clarity and consistency. Signed-off-by: appleboy <appleboy.tw@gmail.com> * test: improve initialization and labeling of Prometheus metrics - Add a test to verify that Prometheus metrics initialize correctly with extra labels. - Ensure metrics variables are properly initialized and not nil. - Check that WithLabelValues accepts both default and extra labels without causing a panic. Signed-off-by: appleboy <appleboy.tw@gmail.com> * test: improve Prometheus metrics testing for extra label handling - Remove a redundant test for WithLabelValues label length. - Add a new test to verify that extraLabels are correctly included in exported Prometheus metrics. Signed-off-by: appleboy <appleboy.tw@gmail.com> * refactor: refactor metrics to support custom Prometheus registries - Refactor metrics initialization to accept a custom Prometheus registry, defaulting to the global registry when nil - Replace promauto with direct metric construction and explicit registration - Update tests to use dedicated, isolated registries instead of the default global registry Signed-off-by: appleboy <appleboy.tw@gmail.com> * Revert README.md to a previous version * docs: document support for custom metric labels in endpoints - Add documentation section explaining support for custom labels on metrics - Provide YAML configuration example illustrating the new labels field for endpoints - Update table of contents to include the custom labels section Signed-off-by: appleboy <appleboy.tw@gmail.com> --------- Signed-off-by: appleboy <appleboy.tw@gmail.com> Signed-off-by: Bo-Yi Wu <appleboy.tw@gmail.com> Co-authored-by: TwiN <twin@linux.com>
This commit is contained in:
Bo-Yi Wu
@@ -3,16 +3,14 @@ package metrics
|
||||
import (
|
||||
"strconv"
|
||||
|
||||
"github.com/TwiN/gatus/v5/config"
|
||||
"github.com/TwiN/gatus/v5/config/endpoint"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
const namespace = "gatus" // The prefix of the metrics
|
||||
|
||||
var (
|
||||
initializedMetrics bool // Whether the metrics have been initialized
|
||||
|
||||
resultTotal *prometheus.CounterVec
|
||||
resultDurationSeconds *prometheus.GaugeVec
|
||||
resultConnectedTotal *prometheus.CounterVec
|
||||
@@ -21,64 +19,79 @@ var (
|
||||
resultEndpointSuccess *prometheus.GaugeVec
|
||||
)
|
||||
|
||||
func initializePrometheusMetrics() {
|
||||
resultTotal = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
func InitializePrometheusMetrics(cfg *config.Config, reg prometheus.Registerer) {
|
||||
if reg == nil {
|
||||
reg = prometheus.DefaultRegisterer
|
||||
}
|
||||
extraLabels := cfg.GetUniqueExtraMetricLabels()
|
||||
resultTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "results_total",
|
||||
Help: "Number of results per endpoint",
|
||||
}, []string{"key", "group", "name", "type", "success"})
|
||||
resultDurationSeconds = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
}, append([]string{"key", "group", "name", "type", "success"}, extraLabels...))
|
||||
reg.MustRegister(resultTotal)
|
||||
resultDurationSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Name: "results_duration_seconds",
|
||||
Help: "Duration of the request in seconds",
|
||||
}, []string{"key", "group", "name", "type"})
|
||||
resultConnectedTotal = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
}, append([]string{"key", "group", "name", "type"}, extraLabels...))
|
||||
reg.MustRegister(resultDurationSeconds)
|
||||
resultConnectedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "results_connected_total",
|
||||
Help: "Total number of results in which a connection was successfully established",
|
||||
}, []string{"key", "group", "name", "type"})
|
||||
resultCodeTotal = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
}, append([]string{"key", "group", "name", "type"}, extraLabels...))
|
||||
reg.MustRegister(resultConnectedTotal)
|
||||
resultCodeTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: "results_code_total",
|
||||
Help: "Total number of results by code",
|
||||
}, []string{"key", "group", "name", "type", "code"})
|
||||
resultCertificateExpirationSeconds = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
}, append([]string{"key", "group", "name", "type", "code"}, extraLabels...))
|
||||
reg.MustRegister(resultCodeTotal)
|
||||
resultCertificateExpirationSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Name: "results_certificate_expiration_seconds",
|
||||
Help: "Number of seconds until the certificate expires",
|
||||
}, []string{"key", "group", "name", "type"})
|
||||
resultEndpointSuccess = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
}, append([]string{"key", "group", "name", "type"}, extraLabels...))
|
||||
reg.MustRegister(resultCertificateExpirationSeconds)
|
||||
resultEndpointSuccess = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Name: "results_endpoint_success",
|
||||
Help: "Displays whether or not the endpoint was a success",
|
||||
}, []string{"key", "group", "name", "type"})
|
||||
}, append([]string{"key", "group", "name", "type"}, extraLabels...))
|
||||
reg.MustRegister(resultEndpointSuccess)
|
||||
}
|
||||
|
||||
// PublishMetricsForEndpoint publishes metrics for the given endpoint and its result.
|
||||
// These metrics will be exposed at /metrics if the metrics are enabled
|
||||
func PublishMetricsForEndpoint(ep *endpoint.Endpoint, result *endpoint.Result) {
|
||||
if !initializedMetrics {
|
||||
initializePrometheusMetrics()
|
||||
initializedMetrics = true
|
||||
func PublishMetricsForEndpoint(ep *endpoint.Endpoint, result *endpoint.Result, extraLabels []string) {
|
||||
labelValues := []string{}
|
||||
for _, label := range extraLabels {
|
||||
if value, ok := ep.ExtraLabels[label]; ok {
|
||||
labelValues = append(labelValues, value)
|
||||
} else {
|
||||
labelValues = append(labelValues, "")
|
||||
}
|
||||
}
|
||||
|
||||
endpointType := ep.Type()
|
||||
resultTotal.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType), strconv.FormatBool(result.Success)).Inc()
|
||||
resultDurationSeconds.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType)).Set(result.Duration.Seconds())
|
||||
resultTotal.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType), strconv.FormatBool(result.Success)}, labelValues...)...).Inc()
|
||||
resultDurationSeconds.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType)}, labelValues...)...).Set(result.Duration.Seconds())
|
||||
if result.Connected {
|
||||
resultConnectedTotal.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType)).Inc()
|
||||
resultConnectedTotal.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType)}, labelValues...)...).Inc()
|
||||
}
|
||||
if result.DNSRCode != "" {
|
||||
resultCodeTotal.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType), result.DNSRCode).Inc()
|
||||
resultCodeTotal.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType), result.DNSRCode}, labelValues...)...).Inc()
|
||||
}
|
||||
if result.HTTPStatus != 0 {
|
||||
resultCodeTotal.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType), strconv.Itoa(result.HTTPStatus)).Inc()
|
||||
resultCodeTotal.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType), strconv.Itoa(result.HTTPStatus)}, labelValues...)...).Inc()
|
||||
}
|
||||
if result.CertificateExpiration != 0 {
|
||||
resultCertificateExpirationSeconds.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType)).Set(result.CertificateExpiration.Seconds())
|
||||
resultCertificateExpirationSeconds.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType)}, labelValues...)...).Set(result.CertificateExpiration.Seconds())
|
||||
}
|
||||
if result.Success {
|
||||
resultEndpointSuccess.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType)).Set(1)
|
||||
resultEndpointSuccess.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType)}, labelValues...)...).Set(1)
|
||||
} else {
|
||||
resultEndpointSuccess.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType)).Set(0)
|
||||
resultEndpointSuccess.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType)}, labelValues...)...).Set(0)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user