fix(metrics): improve Prometheus metrics registration (#1186)

feat: improve Prometheus metrics registration and cleanup

- Add a function to unregister all previously registered Prometheus metrics
- Track metric initialization state to prevent duplicate registration
- Ensure metrics are unregistered before re-initializing them
- Store the current registerer for proper metric cleanup
- Call the new unregister function during application stop

ref: https://github.com/TwiN/gatus/pull/979#issuecomment-3157044249

Signed-off-by: Bo-Yi Wu <appleboy.tw@gmail.com>

main.go

@@ -58,6 +58,7 @@ func start(cfg *config.Config) {
 func stop(cfg *config.Config) {
 	watchdog.Shutdown(cfg)
 	controller.Shutdown()
+	metrics.UnregisterPrometheusMetrics()
 }
 
 func save() {

metrics/metrics.go

@@ -17,12 +17,55 @@ var (
 	resultCodeTotal                    *prometheus.CounterVec
 	resultCertificateExpirationSeconds *prometheus.GaugeVec
 	resultEndpointSuccess              *prometheus.GaugeVec
+
+	// Track if metrics have been initialized to prevent duplicate registration
+	metricsInitialized bool
+	currentRegisterer  prometheus.Registerer
 )
 
+// UnregisterPrometheusMetrics unregisters all previously registered metrics
+func UnregisterPrometheusMetrics() {
+	if !metricsInitialized || currentRegisterer == nil {
+		return
+	}
+
+	// Unregister all metrics if they exist
+	if resultTotal != nil {
+		currentRegisterer.Unregister(resultTotal)
+	}
+	if resultDurationSeconds != nil {
+		currentRegisterer.Unregister(resultDurationSeconds)
+	}
+	if resultConnectedTotal != nil {
+		currentRegisterer.Unregister(resultConnectedTotal)
+	}
+	if resultCodeTotal != nil {
+		currentRegisterer.Unregister(resultCodeTotal)
+	}
+	if resultCertificateExpirationSeconds != nil {
+		currentRegisterer.Unregister(resultCertificateExpirationSeconds)
+	}
+	if resultEndpointSuccess != nil {
+		currentRegisterer.Unregister(resultEndpointSuccess)
+	}
+
+	metricsInitialized = false
+	currentRegisterer = nil
+}
+
 func InitializePrometheusMetrics(cfg *config.Config, reg prometheus.Registerer) {
+	// If metrics are already initialized, unregister them first
+	if metricsInitialized {
+		UnregisterPrometheusMetrics()
+	}
+
 	if reg == nil {
 		reg = prometheus.DefaultRegisterer
 	}
+
+	// Store the registerer for later unregistration
+	currentRegisterer = reg
+
 	extraLabels := cfg.GetUniqueExtraMetricLabels()
 	resultTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: namespace,
@@ -30,36 +73,44 @@ func InitializePrometheusMetrics(cfg *config.Config, reg prometheus.Registerer)
 		Help:      "Number of results per endpoint",
 	}, append([]string{"key", "group", "name", "type", "success"}, extraLabels...))
 	reg.MustRegister(resultTotal)
+
 	resultDurationSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: namespace,
 		Name:      "results_duration_seconds",
 		Help:      "Duration of the request in seconds",
 	}, append([]string{"key", "group", "name", "type"}, extraLabels...))
 	reg.MustRegister(resultDurationSeconds)
+
 	resultConnectedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: namespace,
 		Name:      "results_connected_total",
 		Help:      "Total number of results in which a connection was successfully established",
 	}, append([]string{"key", "group", "name", "type"}, extraLabels...))
 	reg.MustRegister(resultConnectedTotal)
+
 	resultCodeTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: namespace,
 		Name:      "results_code_total",
 		Help:      "Total number of results by code",
 	}, append([]string{"key", "group", "name", "type", "code"}, extraLabels...))
 	reg.MustRegister(resultCodeTotal)
+
 	resultCertificateExpirationSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: namespace,
 		Name:      "results_certificate_expiration_seconds",
 		Help:      "Number of seconds until the certificate expires",
 	}, append([]string{"key", "group", "name", "type"}, extraLabels...))
 	reg.MustRegister(resultCertificateExpirationSeconds)
+
 	resultEndpointSuccess = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: namespace,
 		Name:      "results_endpoint_success",
 		Help:      "Displays whether or not the endpoint was a success",
 	}, append([]string{"key", "group", "name", "type"}, extraLabels...))
 	reg.MustRegister(resultEndpointSuccess)
+
+	// Mark as initialized
+	metricsInitialized = true
 }
 
 // PublishMetricsForEndpoint publishes metrics for the given endpoint and its result.