perf: Cap RANDOM_STRING_N to 8182 (#1193)

Bumps [modernc.org/sqlite](https://gitlab.com/cznic/sqlite) from 1.38.0 to 1.38.2.
- [Commits](https://gitlab.com/cznic/sqlite/compare/v1.38.0...v1.38.2)

---
updated-dependencies:
- dependency-name: modernc.org/sqlite
  dependency-version: 1.38.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: TwiN <twin@linux.com>

.examples/nixos/README.md

@@ -0,0 +1,4 @@
+# NixOS
+
+Gatus is implemented as a NixOS module. See [gatus.nix](./gatus.nix) for example
+usage.

.examples/nixos/gatus.nix

@@ -0,0 +1,23 @@
+{
+  services.gatus = {
+    enable = true;
+
+    settings = {
+      web.port = 8080;
+
+      endpoints = [
+        {
+          name = "website";
+          url = "https://twin.sh/health";
+          interval = "5m";
+
+          conditions = [
+            "[STATUS] == 200"
+            "[BODY].status == UP"
+            "[RESPONSE_TIME] < 300"
+          ];
+        }
+      ];
+    };
+  };
+}

.github/codecov.yml

@@ -7,6 +7,6 @@ coverage:
     patch: off
     project:
       default:
-        target: 75%
+        target: 70%
         threshold: null
 

.github/workflows/benchmark.yml

@@ -18,7 +18,7 @@ jobs:
   build:
     name: benchmark
     runs-on: ubuntu-latest
-    timeout-minutes: 5
+    timeout-minutes: 15
     steps:
       - uses: actions/setup-go@v5
         with:

.github/workflows/labeler.yml

@@ -26,6 +26,8 @@ jobs:
             gh issue edit "$NUMBER" --add-label "feature"
           elif [[ $TITLE == "fix"* ]]; then
             gh issue edit "$NUMBER" --add-label "bug"
+          elif [[ $TITLE == "docs"* ]]; then
+            gh issue edit "$NUMBER" --add-label "documentation"
           fi
           if [[ $TITLE == *"alerting"* || $TITLE == *"provider"* || $TITLE == *"alert"* ]]; then
             gh issue edit "$NUMBER" --add-label "area/alerting"
@@ -39,4 +41,3 @@ jobs:
           if [[ $TITLE == *"metric"* || $TITLE == *"prometheus"* ]]; then
             gh issue edit "$NUMBER" --add-label "area/metrics"
           fi
-          

.github/workflows/publish-custom.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   publish-custom:
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
       - name: Set up QEMU

.github/workflows/publish-experimental.yml

@@ -3,7 +3,7 @@ on: [workflow_dispatch]
 jobs:
   publish-experimental:
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
       - name: Set up QEMU

.github/workflows/publish-latest.yml

@@ -11,7 +11,7 @@ jobs:
   publish-latest:
     runs-on: ubuntu-latest
     if: ${{ (github.event.workflow_run.conclusion == 'success') && (github.event.workflow_run.head_repository.full_name == github.repository) }}
-    timeout-minutes: 90
+    timeout-minutes: 240
     steps:
       - uses: actions/checkout@v4
       - name: Set up QEMU

.github/workflows/publish-release.yml

@@ -6,7 +6,7 @@ jobs:
   publish-release:
     name: publish-release
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    timeout-minutes: 240
     steps:
       - uses: actions/checkout@v4
       - name: Set up QEMU

.github/workflows/test-ui.yml

@@ -11,7 +11,7 @@ on:
 jobs:
   test-ui:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
       - run: make frontend-install-dependencies

.github/workflows/test.yml

@@ -14,7 +14,7 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 30
     steps:
       - uses: actions/setup-go@v5
         with:

README.md

@@ -59,6 +59,8 @@ Have any feedback or questions? [Create a discussion](https://github.com/TwiN/ga
     - [Configuring GitLab alerts](#configuring-gitlab-alerts)
     - [Configuring Google Chat alerts](#configuring-google-chat-alerts)
     - [Configuring Gotify alerts](#configuring-gotify-alerts)
+    - [Configuring HomeAssistant alerts](#configuring-homeassistant-alerts)
+    - [Configuring Ilert alerts](#configuring-ilert-alerts)
     - [Configuring Incident.io alerts](#configuring-incidentio-alerts)
     - [Configuring JetBrains Space alerts](#configuring-jetbrains-space-alerts)
     - [Configuring Matrix alerts](#configuring-matrix-alerts)
@@ -82,6 +84,7 @@ Have any feedback or questions? [Create a discussion](https://github.com/TwiN/ga
     - [OIDC](#oidc)
   - [TLS Encryption](#tls-encryption)
   - [Metrics](#metrics)
+    - [Custom Labels](#custom-labels)
   - [Connectivity](#connectivity)
   - [Remote instances (EXPERIMENTAL)](#remote-instances-experimental)
 - [Deployment](#deployment)
@@ -121,6 +124,7 @@ Have any feedback or questions? [Create a discussion](https://github.com/TwiN/ga
     - [Response time (chart)](#response-time-chart)
       - [How to change the color thresholds of the response time badge](#how-to-change-the-color-thresholds-of-the-response-time-badge)
   - [API](#api)
+    - [Interacting with the API programmatically](#interacting-with-the-api-programmatically)
     - [Raw Data](#raw-data)
       - [Uptime](#uptime-1)
       - [Response Time](#response-time-1)
@@ -286,6 +290,13 @@ You can then configure alerts to be triggered when an endpoint is unhealthy once
 | `endpoints[].ui.dont-resolve-failed-conditions` | Whether to resolve failed conditions for the UI.                                                                                            | `false`                    |
 | `endpoints[].ui.badge.response-time`            | List of response time thresholds. Each time a threshold is reached, the badge has a different color.                                        | `[50, 200, 300, 500, 750]` |
 
+You may use the following placeholders in the body (`endpoints[].body`):
+- `[ENDPOINT_NAME]` (resolved from `endpoints[].name`)
+- `[ENDPOINT_GROUP]` (resolved from `endpoints[].group`)
+- `[ENDPOINT_URL]` (resolved from `endpoints[].url`)
+- `[LOCAL_ADDRESS]` (resolves to the local IP and port like `192.0.2.1:25` or `[2001:db8::1]:80`)
+- `[RANDOM_STRING_N]` (resolves to a random string of numbers and letters of length N (max: 8192))
+
 
 ### External Endpoints
 Unlike regular endpoints, external endpoints are not monitored by Gatus, but they are instead pushed programmatically.
@@ -296,14 +307,16 @@ For instance:
 - You can monitor services that are not supported by Gatus
 - You can implement your own monitoring system while using Gatus as the dashboard
 
-| Parameter                      | Description                                                                                                            | Default       |
-|:-------------------------------|:-----------------------------------------------------------------------------------------------------------------------|:--------------|
-| `external-endpoints`           | List of endpoints to monitor.                                                                                          | `[]`          |
-| `external-endpoints[].enabled` | Whether to monitor the endpoint.                                                                                       | `true`        |
-| `external-endpoints[].name`    | Name of the endpoint. Can be anything.                                                                                 | Required `""` |
-| `external-endpoints[].group`   | Group name. Used to group multiple endpoints together on the dashboard. <br />See [Endpoint groups](#endpoint-groups). | `""`          |
-| `external-endpoints[].token`   | Bearer token required to push status to.                                                                               | Required `""` |
-| `external-endpoints[].alerts`  | List of all alerts for a given endpoint. <br />See [Alerting](#alerting).                                              | `[]`          |
+| Parameter                                 | Description                                                                                                                       | Default        |
+|:------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------|:---------------|
+| `external-endpoints`                      | List of endpoints to monitor.                                                                                                     | `[]`           |
+| `external-endpoints[].enabled`            | Whether to monitor the endpoint.                                                                                                  | `true`         |
+| `external-endpoints[].name`               | Name of the endpoint. Can be anything.                                                                                            | Required `""`  |
+| `external-endpoints[].group`              | Group name. Used to group multiple endpoints together on the dashboard. <br />See [Endpoint groups](#endpoint-groups).            | `""`           |
+| `external-endpoints[].token`              | Bearer token required to push status to.                                                                                          | Required `""`  |
+| `external-endpoints[].alerts`             | List of all alerts for a given endpoint. <br />See [Alerting](#alerting).                                                         | `[]`           |
+| `external-endpoints[].heartbeat`          | Heartbeat configuration for monitoring when the external endpoint stops sending updates.                                          | `{}`           |
+| `external-endpoints[].heartbeat.interval` | Expected interval between updates. If no update is received within this interval, alerts will be triggered. Must be at least 10s. | `0` (disabled) |
 
 Example:
 ```yaml
@@ -311,6 +324,8 @@ external-endpoints:
   - name: ext-ep-test
     group: core
     token: "potato"
+    heartbeat:
+      interval: 30m  # Automatically create a failure if no update is received within 30 minutes
     alerts:
       - type: discord
         description: "healthcheck failed"
@@ -319,13 +334,14 @@ external-endpoints:
 
 To push the status of an external endpoint, the request would have to look like this:
 ```
-POST /api/v1/endpoints/{key}/external?success={success}&error={error}
+POST /api/v1/endpoints/{key}/external?success={success}&error={error}&duration={duration}
 ```
 Where:
 - `{key}` has the pattern `<GROUP_NAME>_<ENDPOINT_NAME>` in which both variables have ` `, `/`, `_`, `,`, `.` and `#` replaced by `-`.
   - Using the example configuration above, the key would be `core_ext-ep-test`.
 - `{success}` is a boolean (`true` or `false`) value indicating whether the health check was successful or not.
-- `{error}`: a string describing the reason for a failed health check. If {success} is false, this should contain the error message; if the check is successful, it can be omitted or left empty.
+- `{error}` (optional): a string describing the reason for a failed health check. If {success} is false, this should contain the error message; if the check is successful.
+- `{duration}` (optional): the time that the request took as a duration string (e.g. 10s). 
 
 You must also pass the token as a `Bearer` token in the `Authorization` header.
 
@@ -538,16 +554,17 @@ individual endpoints with configurable descriptions and thresholds.
 
 Alerts are configured at the endpoint level like so:
 
-| Parameter                    | Description                                                                    | Default       |
-|:-----------------------------|:-------------------------------------------------------------------------------|:--------------|
-| `alerts`                     | List of all alerts for a given endpoint.                                       | `[]`          |
-| `alerts[].type`              | Type of alert. <br />See table below for all valid types.                      | Required `""` |
-| `alerts[].enabled`           | Whether to enable the alert.                                                   | `true`        |
-| `alerts[].failure-threshold` | Number of failures in a row needed before triggering the alert.                | `3`           |
-| `alerts[].success-threshold` | Number of successes in a row before an ongoing incident is marked as resolved. | `2`           |
-| `alerts[].send-on-resolved`  | Whether to send a notification once a triggered alert is marked as resolved.   | `false`       |
-| `alerts[].description`       | Description of the alert. Will be included in the alert sent.                  | `""`          |
-| `alerts[].provider-override` | Alerting provider configuration override for the given alert type              | `{}`          |
+| Parameter                            | Description                                                                    | Default       |
+|:-------------------------------------|:-------------------------------------------------------------------------------|:--------------|
+| `alerts`                             | List of all alerts for a given endpoint.                                       | `[]`          |
+| `alerts[].type`                      | Type of alert. <br />See table below for all valid types.                      | Required `""` |
+| `alerts[].enabled`                   | Whether to enable the alert.                                                   | `true`        |
+| `alerts[].failure-threshold`         | Number of failures in a row needed before triggering the alert.                | `3`           |
+| `alerts[].success-threshold`         | Number of successes in a row before an ongoing incident is marked as resolved. | `2`           |
+| `alerts[].minimum-reminder-interval` | Configuration for setting an interval between reminders. (e.g. 30m, 24h)       | `""`          |
+| `alerts[].send-on-resolved`          | Whether to send a notification once a triggered alert is marked as resolved.   | `false`       |
+| `alerts[].description`               | Description of the alert. Will be included in the alert sent.                  | `""`          |
+| `alerts[].provider-override`         | Alerting provider configuration override for the given alert type              | `{}`          |
 
 Here's an example of what an alert configuration might look like at the endpoint level:
 ```yaml
@@ -589,6 +606,7 @@ endpoints:
 | `alerting.gitlab`          | Configuration for alerts of type `gitlab`. <br />See [Configuring GitLab alerts](#configuring-gitlab-alerts).                           | `{}`    |
 | `alerting.googlechat`      | Configuration for alerts of type `googlechat`. <br />See [Configuring Google Chat alerts](#configuring-google-chat-alerts).             | `{}`    |
 | `alerting.gotify`          | Configuration for alerts of type `gotify`. <br />See [Configuring Gotify alerts](#configuring-gotify-alerts).                           | `{}`    |
+| `alerting.ilert`           | Configuration for alerts of type `ilert`. <br />See [Configuring ilert alerts](#configuring-ilert-alerts).                              | `{}`    |
 | `alerting.incident-io`     | Configuration for alerts of type `incident-io`. <br />See [Configuring Incident.io alerts](#configuring-incidentio-alerts).             | `{}`    |
 | `alerting.jetbrainsspace`  | Configuration for alerts of type `jetbrainsspace`. <br />See [Configuring JetBrains Space alerts](#configuring-jetbrains-space-alerts). | `{}`    |
 | `alerting.matrix`          | Configuration for alerts of type `matrix`. <br />See [Configuring Matrix alerts](#configuring-matrix-alerts).                           | `{}`    |
@@ -604,6 +622,7 @@ endpoints:
 | `alerting.telegram`        | Configuration for alerts of type `telegram`. <br />See [Configuring Telegram alerts](#configuring-telegram-alerts).                     | `{}`    |
 | `alerting.twilio`          | Settings for alerts of type `twilio`. <br />See [Configuring Twilio alerts](#configuring-twilio-alerts).                                | `{}`    |
 | `alerting.zulip`           | Configuration for alerts of type `zulip`. <br />See [Configuring Zulip alerts](#configuring-zulip-alerts).                              | `{}`    |
+| `alerting.homeassistant`   | Configuration for alerts of type `homeassistant`. <br />See [Configuring HomeAssistant alerts](#configuring-homeassistant-alerts).        | `{}`    |
 
 
 #### Configuring AWS SES alerts
@@ -895,6 +914,51 @@ endpoints:
 | `alerting.gotify.title`                       | Title of the notification                                                                   | `"Gatus: <endpoint>"` |
 | `alerting.gotify.default-alert`               | Default alert configuration. <br />See [Setting a default alert](#setting-a-default-alert). | N/A                   |
 
+#### Configuring ilert alerts
+| Parameter                              | Description                                                                                | Default |
+|:---------------------------------------|:-------------------------------------------------------------------------------------------|:--------|
+| `alerting.ilert`                   | Configuration for alerts of type `ilert`                                               | `{}`    |
+| `alerting.ilert.integration-key`   | ilert Alert Source integration key                                                    | `""`    |
+| `alerting.ilert.default-alert`     | Default alert configuration. <br />See [Setting a default alert](#setting-a-default-alert) | N/A     |
+| `alerting.ilert.overrides`         | List of overrides that may be prioritized over the default configuration                   | `[]`    |
+| `alerting.ilert.overrides[].group` | Endpoint group for which the configuration will be overridden by this configuration        | `""`    |
+| `alerting.ilert.overrides[].*`     | See `alerting.ilert.*` parameters                                                      | `{}`    |
+
+It is highly recommended to set `endpoints[].alerts[].send-on-resolved` to `true` for alerts
+of type `ilert`, because unlike other alerts, the operation resulting from setting said
+parameter to `true` will not create another alert but mark the alert as resolved on
+ilert instead.
+
+Behavior:
+- By default, `alerting.ilert.integration-key` is used as the integration key
+- If the endpoint being evaluated belongs to a group (`endpoints[].group`) matching the value of `alerting.ilert.overrides[].group`, the provider will use that override's integration key instead of `alerting.ilert.integration-key`'s
+
+```yaml
+alerting:
+  ilert:
+    integration-key: "********************************"
+    # You can also add group-specific integration keys, which will
+    # override the integration key above for the specified groups
+    overrides:
+      - group: "core"
+        integration-key: "********************************"
+
+endpoints:
+  - name: website
+    url: "https://twin.sh/health"
+    interval: 30s
+    conditions:
+      - "[STATUS] == 200"
+      - "[BODY].status == UP"
+      - "[RESPONSE_TIME] < 300"
+    alerts:
+      - type: ilert
+        failure-threshold: 3
+        success-threshold: 5
+        send-on-resolved: true
+        description: "healthcheck failed"
+```
+
 ```yaml
 alerting:
   gotify:
@@ -920,6 +984,75 @@ Here's an example of what the notifications look like:
 ![Gotify notifications](.github/assets/gotify-alerts.png)
 
 
+#### Configuring HomeAssistant alerts
+To configure HomeAssistant alerts, you'll need to add the following to your configuration file:
+
+```yaml
+alerting:
+  homeassistant:
+    url: "http://homeassistant:8123"  # URL of your HomeAssistant instance
+    token: "YOUR_LONG_LIVED_ACCESS_TOKEN"  # Long-lived access token from HomeAssistant
+
+endpoints:
+  - name: my-service
+    url: "https://my-service.com"
+    interval: 5m
+    conditions:
+      - "[STATUS] == 200"
+    alerts:
+      - type: homeassistant
+        enabled: true
+        send-on-resolved: true
+        description: "My service health check"
+        failure-threshold: 3
+        success-threshold: 2
+```
+
+The alerts will be sent as events to HomeAssistant with the event type `gatus_alert`. The event data includes:
+- `status`: "triggered" or "resolved"
+- `endpoint`: The name of the monitored endpoint
+- `description`: The alert description if provided
+- `conditions`: List of conditions and their results
+- `failure_count`: Number of consecutive failures (when triggered)
+- `success_count`: Number of consecutive successes (when resolved)
+
+You can use these events in HomeAssistant automations to:
+- Send notifications
+- Control devices
+- Trigger scenes
+- Log to history
+- And more
+
+Example HomeAssistant automation:
+```yaml
+automation:
+  - alias: "Gatus Alert Handler"
+    trigger:
+      platform: event
+      event_type: gatus_alert
+    action:
+      - service: notify.notify
+        data_template:
+          title: "Gatus Alert: {{ trigger.event.data.endpoint }}"
+          message: >
+            Status: {{ trigger.event.data.status }}
+            {% if trigger.event.data.description %}
+            Description: {{ trigger.event.data.description }}
+            {% endif %}
+            {% for condition in trigger.event.data.conditions %}
+            {{ '✅' if condition.success else '❌' }} {{ condition.condition }}
+            {% endfor %}
+```
+
+To get your HomeAssistant long-lived access token:
+1. Open HomeAssistant
+2. Click on your profile name (bottom left)
+3. Scroll down to "Long-Lived Access Tokens"
+4. Click "Create Token"
+5. Give it a name (e.g., "Gatus")
+6. Copy the token - you'll only see it once!
+
+
 #### Configuring Incident.io alerts
 | Parameter                                | Description                                                                                | Default       |
 |:-----------------------------------------|:-------------------------------------------------------------------------------------------|:--------------|
@@ -1432,6 +1565,7 @@ Here's an example of what the notifications look like:
 | `alerting.telegram`                   | Configuration for alerts of type `telegram`                                                | `{}`                       |
 | `alerting.telegram.token`             | Telegram Bot Token                                                                         | Required `""`              |
 | `alerting.telegram.id`                | Telegram User ID                                                                           | Required `""`              |
+| `alerting.telegram.topic-id`          | Telegram Topic ID in a group corresponds to `message_thread_id` in the Telegram API        | `""`                       |    
 | `alerting.telegram.api-url`           | Telegram API URL                                                                           | `https://api.telegram.org` |
 | `alerting.telegram.client`            | Client configuration. <br />See [Client configuration](#client-configuration).             | `{}`                       |
 | `alerting.telegram.default-alert`     | Default alert configuration. <br />See [Setting a default alert](#setting-a-default-alert) | N/A                        |
@@ -1444,6 +1578,7 @@ alerting:
   telegram:
     token: "123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11"
     id: "0123456789"
+    topic-id: "7"
 
 endpoints:
   - name: website
@@ -1817,6 +1952,23 @@ endpoint on the same port your application is configured to run on (`web.port`).
 
 See [examples/docker-compose-grafana-prometheus](.examples/docker-compose-grafana-prometheus) for further documentation as well as an example.
 
+#### Custom Labels
+
+Added a Labels field to the Config and Endpoint structs to support key-value pairs for metrics. Updated the Prometheus metrics initialization to include dynamic labels from the configuration. See the example below:
+
+```yaml
+endpoints:
+  - name: front-end
+    group: core
+    url: "https://twin.sh/health"
+    interval: 5m
+    conditions:
+      - "[STATUS] == 200"
+      - "[BODY].status == UP"
+      - "[RESPONSE_TIME] < 150"
+    labels:
+      environment: staging
+```
 
 ### Connectivity
 | Parameter                       | Description                                | Default       |
@@ -2004,8 +2156,9 @@ endpoints:
     conditions:
       - "[CONNECTED] == true"
 ```
+If `endpoints[].body` is set then it is sent and the first 1024 bytes of the response will be in `[BODY]`.
 
-Placeholders `[STATUS]` and `[BODY]` as well as the fields `endpoints[].body`, `endpoints[].headers`,
+Placeholder `[STATUS]` as well as the fields `endpoints[].headers`,
 `endpoints[].method` and `endpoints[].graphql` are not supported for TCP endpoints.
 
 This works for applications such as databases (Postgres, MySQL, etc.) and caches (Redis, Memcached, etc.).
@@ -2025,7 +2178,9 @@ endpoints:
       - "[CONNECTED] == true"
 ```
 
-Placeholders `[STATUS]` and `[BODY]` as well as the fields `endpoints[].body`, `endpoints[].headers`,
+If `endpoints[].body` is set then it is sent and the first 1024 bytes of the response will be in `[BODY]`.
+
+Placeholder `[STATUS]` as well as the fields `endpoints[].headers`,
 `endpoints[].method` and `endpoints[].graphql` are not supported for UDP endpoints.
 
 This works for UDP based application.
@@ -2060,7 +2215,8 @@ endpoints:
 ```
 
 The `[BODY]` placeholder contains the output of the query, and `[CONNECTED]`
-shows whether the connection was successfully established.
+shows whether the connection was successfully established. You can use Go template 
+syntax. The functions LocalAddr and RandomString with a length can be used.
 
 
 ### Monitoring an endpoint using ICMP
@@ -2172,6 +2328,11 @@ endpoints:
       - "[CERTIFICATE_EXPIRATION] > 48h"
 ```
 
+If `endpoints[].body` is set then it is sent and the first 1024 bytes of the response will be in `[BODY]`.
+
+Placeholder `[STATUS]` as well as the fields `endpoints[].headers`,
+`endpoints[].method` and `endpoints[].graphql` are not supported for TLS endpoints.
+
 
 ### Monitoring domain expiration
 You can monitor the expiration of a domain with all endpoint types except for DNS by using the `[DOMAIN_EXPIRATION]`
@@ -2500,6 +2661,11 @@ Gzip compression will be used if the `Accept-Encoding` HTTP header contains `gzi
 The API will return a JSON payload with the `Content-Type` response header set to `application/json`.
 No such header is required to query the API.
 
+
+#### Interacting with the API programmatically
+See [TwiN/gatus-sdk](https://github.com/TwiN/gatus-sdk)
+
+
 #### Raw Data
 Gatus exposes the raw data for one of your monitored endpoints.
 This allows you to track and aggregate data in your own applications for monitored endpoints. For instance if you want to track uptime for a period longer than 7 days.
@@ -2532,6 +2698,7 @@ For instance, if you want the raw response time data for the last 24 hours from
 https://example.com/api/v1/endpoints/core_frontend/response-times/24h
 ```
 
+
 ### Installing as binary
 You can download Gatus as a binary using the following command:
 ```

alerting/alert/alert.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/TwiN/logr"
 	"gopkg.in/yaml.v3"
@@ -35,6 +36,9 @@ type Alert struct {
 	// SuccessThreshold defines how many successful executions must happen in a row before an ongoing incident is marked as resolved
 	SuccessThreshold int `yaml:"success-threshold"`
 
+	// MinimumReminderInterval is the interval between reminders
+	MinimumReminderInterval time.Duration `yaml:"minimum-reminder-interval,omitempty"`
+
 	// Description of the alert. Will be included in the alert sent.
 	//
 	// This is a pointer, because it is populated by YAML and we need to know whether it was explicitly set to a value

alerting/alert/type.go

@@ -32,6 +32,12 @@ const (
 	// TypeGotify is the Type for the gotify alerting provider
 	TypeGotify Type = "gotify"
 
+  // TypeHomeAssistant is the Type for the homeassistant alerting provider
+	TypeHomeAssistant Type = "homeassistant"
+  
+	// TypeIlert is the Type for the ilert alerting provider
+	TypeIlert Type = "ilert"
+
 	// TypeIncidentIO is the Type for the incident-io alerting provider
 	TypeIncidentIO Type = "incident-io"
 

alerting/config.go

@@ -15,6 +15,8 @@ import (
 	"github.com/TwiN/gatus/v5/alerting/provider/gitlab"
 	"github.com/TwiN/gatus/v5/alerting/provider/googlechat"
 	"github.com/TwiN/gatus/v5/alerting/provider/gotify"
+  "github.com/TwiN/gatus/v5/alerting/provider/homeassistant"
+  "github.com/TwiN/gatus/v5/alerting/provider/ilert"
 	"github.com/TwiN/gatus/v5/alerting/provider/incidentio"
 	"github.com/TwiN/gatus/v5/alerting/provider/jetbrainsspace"
 	"github.com/TwiN/gatus/v5/alerting/provider/matrix"
@@ -61,6 +63,12 @@ type Config struct {
 
 	// Gotify is the configuration for the gotify alerting provider
 	Gotify *gotify.AlertProvider `yaml:"gotify,omitempty"`
+  
+	// HomeAssistant is the configuration for the homeassistant alerting provider
+	HomeAssistant *homeassistant.AlertProvider `yaml:"homeassistant,omitempty"`
+  
+	// Ilert is the configuration for the ilert alerting provider
+	Ilert *ilert.AlertProvider `yaml:"ilert,omitempty"`
 
 	// IncidentIO is the configuration for the incident-io alerting provider
 	IncidentIO *incidentio.AlertProvider `yaml:"incident-io,omitempty"`

alerting/provider/homeassistant/homeassistant.go

@@ -0,0 +1,196 @@
+package homeassistant
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/TwiN/gatus/v5/alerting/alert"
+	"github.com/TwiN/gatus/v5/client"
+	"github.com/TwiN/gatus/v5/config/endpoint"
+	"gopkg.in/yaml.v3"
+)
+
+var (
+	ErrURLNotSet              = errors.New("url not set")
+	ErrTokenNotSet            = errors.New("token not set")
+	ErrDuplicateGroupOverride = errors.New("duplicate group override")
+)
+
+type Config struct {
+	URL   string `yaml:"url"`
+	Token string `yaml:"token"`
+}
+
+func (cfg *Config) Validate() error {
+	if len(cfg.URL) == 0 {
+		return ErrURLNotSet
+	}
+	if len(cfg.Token) == 0 {
+		return ErrTokenNotSet
+	}
+	return nil
+}
+
+func (cfg *Config) Merge(override *Config) {
+	if len(override.URL) > 0 {
+		cfg.URL = override.URL
+	}
+	if len(override.Token) > 0 {
+		cfg.Token = override.Token
+	}
+}
+
+// AlertProvider is the configuration necessary for sending an alert using HomeAssistant
+type AlertProvider struct {
+	DefaultConfig Config `yaml:",inline"`
+
+	// DefaultAlert is the default alert configuration to use for endpoints with an alert of the appropriate type
+	DefaultAlert *alert.Alert `yaml:"default-alert,omitempty"`
+
+	// Overrides is a list of Override that may be prioritized over the default configuration
+	Overrides []Override `yaml:"overrides,omitempty"`
+}
+
+// Override is a case under which the default integration is overridden
+type Override struct {
+	Group  string `yaml:"group"`
+	Config `yaml:",inline"`
+}
+
+// Validate the provider's configuration
+func (provider *AlertProvider) Validate() error {
+	registeredGroups := make(map[string]bool)
+	if provider.Overrides != nil {
+		for _, override := range provider.Overrides {
+			if isAlreadyRegistered := registeredGroups[override.Group]; isAlreadyRegistered || override.Group == "" {
+				return ErrDuplicateGroupOverride
+			}
+			registeredGroups[override.Group] = true
+		}
+	}
+	return provider.DefaultConfig.Validate()
+}
+
+// Send an alert using the provider
+func (provider *AlertProvider) Send(ep *endpoint.Endpoint, alert *alert.Alert, result *endpoint.Result, resolved bool) error {
+	cfg, err := provider.GetConfig(ep.Group, alert)
+	if err != nil {
+		return err
+	}
+	buffer := bytes.NewBuffer(provider.buildRequestBody(ep, alert, result, resolved))
+	request, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/events/gatus_alert", cfg.URL), buffer)
+	if err != nil {
+		return err
+	}
+	request.Header.Set("Content-Type", "application/json")
+	request.Header.Set("Authorization", "Bearer "+cfg.Token)
+	response, err := client.GetHTTPClient(nil).Do(request)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	if response.StatusCode > 399 {
+		body, _ := io.ReadAll(response.Body)
+		return fmt.Errorf("call to provider alert returned status code %d: %s", response.StatusCode, string(body))
+	}
+	return err
+}
+
+type Body struct {
+	EventType string `json:"event_type"`
+	EventData struct {
+		Status      string `json:"status"`
+		Endpoint    string `json:"endpoint"`
+		Description string `json:"description,omitempty"`
+		Conditions  []struct {
+			Condition string `json:"condition"`
+			Success   bool   `json:"success"`
+		} `json:"conditions,omitempty"`
+		FailureCount int `json:"failure_count,omitempty"`
+		SuccessCount int `json:"success_count,omitempty"`
+	} `json:"event_data"`
+}
+
+// buildRequestBody builds the request body for the provider
+func (provider *AlertProvider) buildRequestBody(ep *endpoint.Endpoint, alert *alert.Alert, result *endpoint.Result, resolved bool) []byte {
+	body := Body{
+		EventType: "gatus_alert",
+		EventData: struct {
+			Status      string `json:"status"`
+			Endpoint    string `json:"endpoint"`
+			Description string `json:"description,omitempty"`
+			Conditions  []struct {
+				Condition string `json:"condition"`
+				Success   bool   `json:"success"`
+			} `json:"conditions,omitempty"`
+			FailureCount int `json:"failure_count,omitempty"`
+			SuccessCount int `json:"success_count,omitempty"`
+		}{
+			Status:   "resolved",
+			Endpoint: ep.DisplayName(),
+		},
+	}
+
+	if !resolved {
+		body.EventData.Status = "triggered"
+		body.EventData.FailureCount = alert.FailureThreshold
+	} else {
+		body.EventData.SuccessCount = alert.SuccessThreshold
+	}
+
+	if alertDescription := alert.GetDescription(); len(alertDescription) > 0 {
+		body.EventData.Description = alertDescription
+	}
+
+	if len(result.ConditionResults) > 0 {
+		for _, conditionResult := range result.ConditionResults {
+			body.EventData.Conditions = append(body.EventData.Conditions, struct {
+				Condition string `json:"condition"`
+				Success   bool   `json:"success"`
+			}{
+				Condition: conditionResult.Condition,
+				Success:   conditionResult.Success,
+			})
+		}
+	}
+
+	bodyAsJSON, _ := json.Marshal(body)
+	return bodyAsJSON
+}
+
+// GetDefaultAlert returns the provider's default alert configuration
+func (provider *AlertProvider) GetDefaultAlert() *alert.Alert {
+	return provider.DefaultAlert
+}
+
+// GetConfig returns the configuration for the provider with the overrides applied
+func (provider *AlertProvider) GetConfig(group string, alert *alert.Alert) (*Config, error) {
+	cfg := provider.DefaultConfig
+	if provider.Overrides != nil {
+		for _, override := range provider.Overrides {
+			if group == override.Group {
+				cfg.Merge(&override.Config)
+				break
+			}
+		}
+	}
+	if len(alert.ProviderOverride) != 0 {
+		overrideConfig := Config{}
+		if err := yaml.Unmarshal(alert.ProviderOverrideAsBytes(), &overrideConfig); err != nil {
+			return nil, err
+		}
+		cfg.Merge(&overrideConfig)
+	}
+	err := cfg.Validate()
+	return &cfg, err
+}
+
+// ValidateOverrides validates the alert's provider override and, if present, the group override
+func (provider *AlertProvider) ValidateOverrides(group string, alert *alert.Alert) error {
+	_, err := provider.GetConfig(group, alert)
+	return err
+}

alerting/provider/homeassistant/homeassistant_test.go

@@ -0,0 +1,158 @@
+package homeassistant
+
+import (
+	"encoding/json"
+	"net/http"
+	"testing"
+
+	"github.com/TwiN/gatus/v5/alerting/alert"
+	"github.com/TwiN/gatus/v5/client"
+	"github.com/TwiN/gatus/v5/config/endpoint"
+	"github.com/TwiN/gatus/v5/test"
+)
+
+func TestAlertProvider_Validate(t *testing.T) {
+	invalidProvider := AlertProvider{DefaultConfig: Config{URL: "", Token: ""}}
+	if err := invalidProvider.Validate(); err == nil {
+		t.Error("provider shouldn't have been valid")
+	}
+	invalidProviderNoToken := AlertProvider{DefaultConfig: Config{URL: "http://homeassistant:8123", Token: ""}}
+	if err := invalidProviderNoToken.Validate(); err == nil {
+		t.Error("provider shouldn't have been valid")
+	}
+	validProvider := AlertProvider{DefaultConfig: Config{URL: "http://homeassistant:8123", Token: "token"}}
+	if err := validProvider.Validate(); err != nil {
+		t.Error("provider should've been valid")
+	}
+}
+
+func TestAlertProvider_ValidateWithOverride(t *testing.T) {
+	providerWithInvalidOverrideGroup := AlertProvider{
+		Overrides: []Override{
+			{
+				Config: Config{URL: "http://homeassistant:8123", Token: "token"},
+				Group:  "",
+			},
+		},
+	}
+	if err := providerWithInvalidOverrideGroup.Validate(); err == nil {
+		t.Error("provider Group shouldn't have been valid")
+	}
+	providerWithValidOverride := AlertProvider{
+		DefaultConfig: Config{URL: "http://homeassistant:8123", Token: "token"},
+		Overrides: []Override{
+			{
+				Config: Config{URL: "http://homeassistant:8123", Token: "token"},
+				Group:  "group",
+			},
+		},
+	}
+	if err := providerWithValidOverride.Validate(); err != nil {
+		t.Error("provider should've been valid")
+	}
+}
+
+func TestAlertProvider_Send(t *testing.T) {
+	defer client.InjectHTTPClient(nil)
+	firstDescription := "description-1"
+	secondDescription := "description-2"
+	scenarios := []struct {
+		Name             string
+		Provider         AlertProvider
+		Alert            alert.Alert
+		Resolved         bool
+		MockRoundTripper test.MockRoundTripper
+		ExpectedError    bool
+	}{
+		{
+			Name:     "triggered",
+			Provider: AlertProvider{DefaultConfig: Config{URL: "http://homeassistant:8123", Token: "token"}},
+			Alert:    alert.Alert{Description: &firstDescription, SuccessThreshold: 5, FailureThreshold: 3},
+			Resolved: false,
+			MockRoundTripper: test.MockRoundTripper(func(r *http.Request) *http.Response {
+				return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody}
+			}),
+			ExpectedError: false,
+		},
+		{
+			Name:     "triggered-error",
+			Provider: AlertProvider{DefaultConfig: Config{URL: "http://homeassistant:8123", Token: "token"}},
+			Alert:    alert.Alert{Description: &firstDescription, SuccessThreshold: 5, FailureThreshold: 3},
+			Resolved: false,
+			MockRoundTripper: test.MockRoundTripper(func(r *http.Request) *http.Response {
+				return &http.Response{StatusCode: http.StatusInternalServerError, Body: http.NoBody}
+			}),
+			ExpectedError: true,
+		},
+		{
+			Name:     "resolved",
+			Provider: AlertProvider{DefaultConfig: Config{URL: "http://homeassistant:8123", Token: "token"}},
+			Alert:    alert.Alert{Description: &secondDescription, SuccessThreshold: 5, FailureThreshold: 3},
+			Resolved: true,
+			MockRoundTripper: test.MockRoundTripper(func(r *http.Request) *http.Response {
+				return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody}
+			}),
+			ExpectedError: false,
+		},
+	}
+	for _, scenario := range scenarios {
+		t.Run(scenario.Name, func(t *testing.T) {
+			client.InjectHTTPClient(&http.Client{Transport: scenario.MockRoundTripper})
+			err := scenario.Provider.Send(
+				&endpoint.Endpoint{Name: "endpoint-name"},
+				&scenario.Alert,
+				&endpoint.Result{
+					ConditionResults: []*endpoint.ConditionResult{
+						{Condition: "SUCCESSFUL_CONDITION", Success: true},
+						{Condition: "FAILING_CONDITION", Success: false},
+					},
+				},
+				scenario.Resolved,
+			)
+			if scenario.ExpectedError && err == nil {
+				t.Error("expected error, got none")
+			}
+			if !scenario.ExpectedError && err != nil {
+				t.Error("expected no error, got", err.Error())
+			}
+		})
+	}
+}
+
+func TestAlertProvider_buildRequestBody(t *testing.T) {
+	description := "test-description"
+	provider := AlertProvider{DefaultConfig: Config{URL: "http://homeassistant:8123", Token: "token"}}
+	body := provider.buildRequestBody(
+		&endpoint.Endpoint{Name: "endpoint-name"},
+		&alert.Alert{Description: &description, SuccessThreshold: 5, FailureThreshold: 3},
+		&endpoint.Result{
+			ConditionResults: []*endpoint.ConditionResult{
+				{Condition: "SUCCESSFUL_CONDITION", Success: true},
+				{Condition: "FAILING_CONDITION", Success: false},
+			},
+		},
+		false,
+	)
+	var decodedBody Body
+	if err := json.Unmarshal(body, &decodedBody); err != nil {
+		t.Error("expected body to be valid JSON, got error:", err.Error())
+	}
+	if decodedBody.EventType != "gatus_alert" {
+		t.Errorf("expected event_type to be gatus_alert, got %s", decodedBody.EventType)
+	}
+	if decodedBody.EventData.Status != "triggered" {
+		t.Errorf("expected status to be triggered, got %s", decodedBody.EventData.Status)
+	}
+	if decodedBody.EventData.Description != description {
+		t.Errorf("expected description to be %s, got %s", description, decodedBody.EventData.Description)
+	}
+	if len(decodedBody.EventData.Conditions) != 2 {
+		t.Errorf("expected 2 conditions, got %d", len(decodedBody.EventData.Conditions))
+	}
+	if !decodedBody.EventData.Conditions[0].Success {
+		t.Error("expected first condition to be successful")
+	}
+	if decodedBody.EventData.Conditions[1].Success {
+		t.Error("expected second condition to be unsuccessful")
+	}
+}

alerting/provider/ilert/ilert.go

@@ -0,0 +1,168 @@
+package ilert
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/TwiN/gatus/v5/alerting/alert"
+	"github.com/TwiN/gatus/v5/client"
+	"github.com/TwiN/gatus/v5/config/endpoint"
+	"gopkg.in/yaml.v3"
+)
+
+const (
+	restAPIUrl = "https://api.ilert.com/api/v1/events/gatus/"
+)
+
+var (
+	ErrIntegrationKeyNotSet   = errors.New("integration key is not set")
+	ErrDuplicateGroupOverride = errors.New("duplicate group override")
+)
+
+type Config struct {
+	IntegrationKey string `yaml:"integration-key"`
+}
+
+func (cfg *Config) Validate() error {
+	if len(cfg.IntegrationKey) == 0 {
+		return ErrIntegrationKeyNotSet
+	}
+	return nil
+}
+
+func (cfg *Config) Merge(override *Config) {
+	if len(override.IntegrationKey) > 0 {
+		cfg.IntegrationKey = override.IntegrationKey
+	}
+}
+
+// AlertProvider is the configuration necessary for sending an alert using ilert
+type AlertProvider struct {
+	DefaultConfig Config `yaml:",inline"`
+
+	// DefaultAlert is the default alert configuration to use for endpoints with an alert of the appropriate type
+	DefaultAlert *alert.Alert `yaml:"default-alert,omitempty"`
+
+	// Overrides is a list of Override that may be prioritized over the default configuration
+	Overrides []Override `yaml:"overrides,omitempty"`
+}
+
+type Override struct {
+	Group  string `yaml:"group"`
+	Config `yaml:",inline"`
+}
+
+func (provider *AlertProvider) Validate() error {
+	registeredGroups := make(map[string]bool)
+	if provider.Overrides != nil {
+		for _, override := range provider.Overrides {
+			if isAlreadyRegistered := registeredGroups[override.Group]; isAlreadyRegistered || override.Group == "" {
+				return ErrDuplicateGroupOverride
+			}
+			registeredGroups[override.Group] = true
+		}
+	}
+	return provider.DefaultConfig.Validate()
+}
+
+func (provider *AlertProvider) Send(ep *endpoint.Endpoint, alert *alert.Alert, result *endpoint.Result, resolved bool) error {
+	cfg, err := provider.GetConfig(ep.Group, alert)
+	if err != nil {
+		return err
+	}
+	buffer := bytes.NewBuffer(provider.buildRequestBody(cfg, ep, alert, result, resolved))
+
+	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s%s", restAPIUrl, cfg.IntegrationKey), buffer)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	response, err := client.GetHTTPClient(nil).Do(req)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	if response.StatusCode > 399 {
+		body, _ := io.ReadAll(response.Body)
+		return fmt.Errorf("call to provider alert returned status code %d: %s", response.StatusCode, string(body))
+	}
+
+	return err
+}
+
+type Body struct {
+	Alert            alert.Alert                 `json:"alert"`
+	Name             string                      `json:"name"`
+	Group            string                      `json:"group"`
+	Status           string                      `json:"status"`
+	Title            string                      `json:"title"`
+	Details          string                      `json:"details,omitempty"`
+	ConditionResults []*endpoint.ConditionResult `json:"condition_results"`
+	URL              string                      `json:"url"`
+}
+
+func (provider *AlertProvider) buildRequestBody(cfg *Config, ep *endpoint.Endpoint, alert *alert.Alert, result *endpoint.Result, resolved bool) []byte {
+	var details, status string
+	if resolved {
+		status = "resolved"
+	} else {
+		status = "firing"
+	}
+
+	if len(alert.GetDescription()) > 0 {
+		details = alert.GetDescription()
+	} else {
+		details = "No description"
+	}
+
+	var body []byte
+	body, _ = json.Marshal(Body{
+		Alert:            *alert,
+		Name:             ep.Name,
+		Group:            ep.Group,
+		Title:            ep.DisplayName(),
+		Status:           status,
+		Details:          details,
+		ConditionResults: result.ConditionResults,
+		URL:              ep.URL,
+	})
+	return body
+}
+
+func (provider *AlertProvider) GetDefaultAlert() *alert.Alert {
+	return provider.DefaultAlert
+}
+
+func (provider *AlertProvider) GetConfig(group string, alert *alert.Alert) (*Config, error) {
+	cfg := provider.DefaultConfig
+	// Handle group overrides
+	if provider.Overrides != nil {
+		for _, override := range provider.Overrides {
+			if group == override.Group {
+				cfg.Merge(&override.Config)
+				break
+			}
+		}
+	}
+	// Handle alert overrides
+	if len(alert.ProviderOverride) != 0 {
+		overrideConfig := Config{}
+		if err := yaml.Unmarshal(alert.ProviderOverrideAsBytes(), &overrideConfig); err != nil {
+			return nil, err
+		}
+		cfg.Merge(&overrideConfig)
+	}
+	// Validate the configuration
+	err := cfg.Validate()
+	return &cfg, err
+}
+
+// ValidateOverrides validates the alert's provider override and, if present, the group override
+func (provider *AlertProvider) ValidateOverrides(group string, alert *alert.Alert) error {
+	_, err := provider.GetConfig(group, alert)
+	return err
+}

alerting/provider/ilert/ilert_test.go

@@ -0,0 +1,322 @@
+package ilert
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"net/http"
+	"testing"
+
+	"github.com/TwiN/gatus/v5/alerting/alert"
+	"github.com/TwiN/gatus/v5/client"
+	"github.com/TwiN/gatus/v5/config/endpoint"
+	"github.com/TwiN/gatus/v5/test"
+)
+
+func TestAlertProvider_Validate(t *testing.T) {
+	scenarios := []struct {
+		name     string
+		provider AlertProvider
+		expected bool
+	}{
+		{
+			name: "valid",
+			provider: AlertProvider{
+				DefaultConfig: Config{
+					IntegrationKey: "some-random-key",
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "invalid-integration-key",
+			provider: AlertProvider{
+				DefaultConfig: Config{
+					IntegrationKey: "",
+				},
+			},
+			expected: false,
+		},
+	}
+	for _, scenario := range scenarios {
+		t.Run(scenario.name, func(t *testing.T) {
+			err := scenario.provider.Validate()
+			if scenario.expected && err != nil {
+				t.Error("expected no error, got", err.Error())
+			}
+			if !scenario.expected && err == nil {
+				t.Error("expected error, got none")
+			}
+		})
+	}
+}
+
+func TestAlertProvider_ValidateWithOverride(t *testing.T) {
+	providerWithInvalidOverrideGroup := AlertProvider{
+		DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000001"},
+		Overrides: []Override{
+			{
+				Config: Config{IntegrationKey: "00000000000000000000000000000002"},
+				Group:  "",
+			},
+		},
+	}
+	if err := providerWithInvalidOverrideGroup.Validate(); err == nil {
+		t.Error("provider Group shouldn't have been valid")
+	}
+	providerWithValidOverride := AlertProvider{
+		DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000001"},
+		Overrides: []Override{
+			{
+				Config: Config{IntegrationKey: "00000000000000000000000000000002"},
+				Group:  "group",
+			},
+		},
+	}
+	if err := providerWithValidOverride.Validate(); err != nil {
+		t.Error("provider should've been valid, got error:", err.Error())
+	}
+}
+
+func TestAlertProvider_Send(t *testing.T) {
+	defer client.InjectHTTPClient(nil)
+	firstDescription := "description-1"
+	secondDescription := "description-2"
+	sendOnResolved := true
+	scenarios := []struct {
+		Name             string
+		Provider         AlertProvider
+		Alert            alert.Alert
+		Resolved         bool
+		MockRoundTripper test.MockRoundTripper
+		ExpectedError    bool
+	}{
+		{
+			Name: "triggered",
+			Provider: AlertProvider{DefaultConfig: Config{
+				IntegrationKey: "some-integration-key",
+			}},
+			Alert:    alert.Alert{Description: &firstDescription, SuccessThreshold: 3, FailureThreshold: 3, ResolveKey: "123", Type: "ilert", SendOnResolved: &sendOnResolved},
+			Resolved: false,
+			MockRoundTripper: test.MockRoundTripper(func(r *http.Request) *http.Response {
+				var b bytes.Buffer
+
+				reader := io.NopCloser(&b)
+				return &http.Response{StatusCode: http.StatusAccepted, Body: reader}
+			}),
+			ExpectedError: false,
+		},
+		{
+			Name: "triggered-error",
+			Provider: AlertProvider{DefaultConfig: Config{
+				IntegrationKey: "some-integration-key",
+			}},
+			Alert:    alert.Alert{Description: &firstDescription, SuccessThreshold: 3, FailureThreshold: 3, ResolveKey: "123", Type: "ilert", SendOnResolved: &sendOnResolved},
+			Resolved: false,
+			MockRoundTripper: test.MockRoundTripper(func(r *http.Request) *http.Response {
+				return &http.Response{StatusCode: http.StatusBadRequest, Body: http.NoBody}
+			}),
+			ExpectedError: true,
+		},
+		{
+			Name: "resolved",
+			Provider: AlertProvider{DefaultConfig: Config{
+				IntegrationKey: "some-integration-key",
+			}},
+			Alert:    alert.Alert{Description: &secondDescription, SuccessThreshold: 3, FailureThreshold: 3, ResolveKey: "123", Type: "ilert", SendOnResolved: &sendOnResolved},
+			Resolved: true,
+			MockRoundTripper: test.MockRoundTripper(func(r *http.Request) *http.Response {
+				var b bytes.Buffer
+				reader := io.NopCloser(&b)
+				return &http.Response{StatusCode: http.StatusAccepted, Body: reader}
+			}),
+			ExpectedError: false,
+		},
+	}
+	for _, scenario := range scenarios {
+		t.Run(scenario.Name, func(t *testing.T) {
+			client.InjectHTTPClient(&http.Client{Transport: scenario.MockRoundTripper})
+			err := scenario.Provider.Send(
+				&endpoint.Endpoint{Name: "endpoint-name"},
+				&scenario.Alert,
+				&endpoint.Result{
+					ConditionResults: []*endpoint.ConditionResult{
+						{Condition: "[CONNECTED] == true", Success: scenario.Resolved},
+						{Condition: "[STATUS] == 200", Success: scenario.Resolved},
+					},
+				},
+				scenario.Resolved,
+			)
+			if scenario.ExpectedError && err == nil {
+				t.Error("expected error, got none")
+			}
+			if !scenario.ExpectedError && err != nil {
+				t.Error("expected no error, got", err.Error())
+			}
+		})
+	}
+}
+
+func TestAlertProvider_BuildRequestBody(t *testing.T) {
+	firstDescription := "description-1"
+	secondDescription := "description-2"
+	sendOnResolved := true
+
+	scenarios := []struct {
+		Name         string
+		Provider     AlertProvider
+		Alert        alert.Alert
+		Resolved     bool
+		ExpectedBody string
+	}{
+		{
+			Name:         "triggered",
+			Provider:     AlertProvider{DefaultConfig: Config{IntegrationKey: "some-integration-key"}},
+			Alert:        alert.Alert{Description: &firstDescription, SuccessThreshold: 3, FailureThreshold: 3, ResolveKey: "123", Type: "ilert", SendOnResolved: &sendOnResolved},
+			Resolved:     false,
+			ExpectedBody: `{"alert":{"Type":"ilert","Enabled":null,"FailureThreshold":3,"SuccessThreshold":3,"MinimumReminderInterval":0,"Description":"description-1","SendOnResolved":true,"ProviderOverride":null,"ResolveKey":"123","Triggered":false},"name":"endpoint-name","group":"","status":"firing","title":"endpoint-name","details":"description-1","condition_results":[{"condition":"[CONNECTED] == true","success":false},{"condition":"[STATUS] == 200","success":false}],"url":""}`,
+		},
+		{
+			Name:         "resolved",
+			Provider:     AlertProvider{DefaultConfig: Config{IntegrationKey: "some-integration-key"}},
+			Alert:        alert.Alert{Description: &firstDescription, SuccessThreshold: 4, FailureThreshold: 3, ResolveKey: "123", Type: "ilert", SendOnResolved: &sendOnResolved},
+			Resolved:     true,
+			ExpectedBody: `{"alert":{"Type":"ilert","Enabled":null,"FailureThreshold":3,"SuccessThreshold":4,"MinimumReminderInterval":0,"Description":"description-1","SendOnResolved":true,"ProviderOverride":null,"ResolveKey":"123","Triggered":false},"name":"endpoint-name","group":"","status":"resolved","title":"endpoint-name","details":"description-1","condition_results":[{"condition":"[CONNECTED] == true","success":true},{"condition":"[STATUS] == 200","success":true}],"url":""}`,
+		},
+		{
+			Name:         "group-override",
+			Provider:     AlertProvider{DefaultConfig: Config{IntegrationKey: "some-integration-key"}, Overrides: []Override{{Group: "g", Config: Config{IntegrationKey: "different-integration-key"}}}},
+			Alert:        alert.Alert{Description: &secondDescription, SuccessThreshold: 5, FailureThreshold: 3, ResolveKey: "123", Type: "ilert", SendOnResolved: &sendOnResolved},
+			Resolved:     false,
+			ExpectedBody: `{"alert":{"Type":"ilert","Enabled":null,"FailureThreshold":3,"SuccessThreshold":5,"MinimumReminderInterval":0,"Description":"description-2","SendOnResolved":true,"ProviderOverride":null,"ResolveKey":"123","Triggered":false},"name":"endpoint-name","group":"","status":"firing","title":"endpoint-name","details":"description-2","condition_results":[{"condition":"[CONNECTED] == true","success":false},{"condition":"[STATUS] == 200","success":false}],"url":""}`,
+		},
+	}
+
+	for _, scenario := range scenarios {
+		t.Run(scenario.Name, func(t *testing.T) {
+			cfg, err := scenario.Provider.GetConfig("g", &scenario.Alert)
+			if err != nil {
+				t.Error("expected no error, got", err.Error())
+			}
+			body := scenario.Provider.buildRequestBody(
+				cfg,
+				&endpoint.Endpoint{Name: "endpoint-name"},
+				&scenario.Alert,
+				&endpoint.Result{
+					ConditionResults: []*endpoint.ConditionResult{
+						{Condition: "[CONNECTED] == true", Success: scenario.Resolved},
+						{Condition: "[STATUS] == 200", Success: scenario.Resolved},
+					},
+				},
+				scenario.Resolved,
+			)
+			if string(body) != scenario.ExpectedBody {
+				t.Errorf("expected:\n%s\ngot:\n%s", scenario.ExpectedBody, body)
+			}
+			out := make(map[string]interface{})
+			if err := json.Unmarshal(body, &out); err != nil {
+				t.Error("expected body to be valid JSON, got error:", err.Error())
+			}
+		})
+	}
+}
+
+func TestAlertProvider_GetDefaultAlert(t *testing.T) {
+	if (&AlertProvider{DefaultAlert: &alert.Alert{}}).GetDefaultAlert() == nil {
+		t.Error("expected default alert to be not nil")
+	}
+	if (&AlertProvider{DefaultAlert: nil}).GetDefaultAlert() != nil {
+		t.Error("expected default alert to be nil")
+	}
+}
+
+func TestAlertProvider_GetConfig(t *testing.T) {
+	scenarios := []struct {
+		Name           string
+		Provider       AlertProvider
+		InputGroup     string
+		InputAlert     alert.Alert
+		ExpectedOutput Config
+	}{
+		{
+			Name: "provider-no-override-specify-no-group-should-default",
+			Provider: AlertProvider{
+				DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000001"},
+				Overrides:     nil,
+			},
+			InputGroup:     "",
+			InputAlert:     alert.Alert{},
+			ExpectedOutput: Config{IntegrationKey: "00000000000000000000000000000001"},
+		},
+		{
+			Name: "provider-no-override-specify-group-should-default",
+			Provider: AlertProvider{
+				DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000001"},
+				Overrides:     nil,
+			},
+			InputGroup:     "group",
+			InputAlert:     alert.Alert{},
+			ExpectedOutput: Config{IntegrationKey: "00000000000000000000000000000001"},
+		},
+		{
+			Name: "provider-with-override-specify-no-group-should-default",
+			Provider: AlertProvider{
+				DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000001"},
+				Overrides: []Override{
+					{
+						Group:  "group",
+						Config: Config{IntegrationKey: "00000000000000000000000000000002"},
+					},
+				},
+			},
+			InputGroup:     "",
+			InputAlert:     alert.Alert{},
+			ExpectedOutput: Config{IntegrationKey: "00000000000000000000000000000001"},
+		},
+		{
+			Name: "provider-with-override-specify-group-should-override",
+			Provider: AlertProvider{
+				DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000001"},
+				Overrides: []Override{
+					{
+						Group:  "group",
+						Config: Config{IntegrationKey: "00000000000000000000000000000002"},
+					},
+				},
+			},
+			InputGroup:     "group",
+			InputAlert:     alert.Alert{},
+			ExpectedOutput: Config{IntegrationKey: "00000000000000000000000000000002"},
+		},
+		{
+			Name: "provider-with-group-override-and-alert-override--alert-override-should-take-precedence",
+			Provider: AlertProvider{
+				DefaultConfig: Config{IntegrationKey: "00000000000000000000000000000001"},
+				Overrides: []Override{
+					{
+						Group:  "group",
+						Config: Config{IntegrationKey: "00000000000000000000000000000002"},
+					},
+				},
+			},
+			InputGroup:     "group",
+			InputAlert:     alert.Alert{ProviderOverride: map[string]any{"integration-key": "00000000000000000000000000000003"}},
+			ExpectedOutput: Config{IntegrationKey: "00000000000000000000000000000003"},
+		},
+	}
+	for _, scenario := range scenarios {
+		t.Run(scenario.Name, func(t *testing.T) {
+			got, err := scenario.Provider.GetConfig(scenario.InputGroup, &scenario.InputAlert)
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+			if got.IntegrationKey != scenario.ExpectedOutput.IntegrationKey {
+				t.Errorf("expected %s, got %s", scenario.ExpectedOutput.IntegrationKey, got.IntegrationKey)
+			}
+			// Test ValidateOverrides as well, since it really just calls GetConfig
+			if err = scenario.Provider.ValidateOverrides(scenario.InputGroup, &scenario.InputAlert); err != nil {
+				t.Errorf("unexpected error: %s", err)
+			}
+		})
+	}
+}

alerting/provider/provider.go

@@ -10,6 +10,9 @@ import (
 	"github.com/TwiN/gatus/v5/alerting/provider/github"
 	"github.com/TwiN/gatus/v5/alerting/provider/gitlab"
 	"github.com/TwiN/gatus/v5/alerting/provider/googlechat"
+  "github.com/TwiN/gatus/v5/alerting/provider/gotify"
+	"github.com/TwiN/gatus/v5/alerting/provider/homeassistant"
+  "github.com/TwiN/gatus/v5/alerting/provider/ilert"
 	"github.com/TwiN/gatus/v5/alerting/provider/incidentio"
 	"github.com/TwiN/gatus/v5/alerting/provider/jetbrainsspace"
 	"github.com/TwiN/gatus/v5/alerting/provider/matrix"
@@ -80,6 +83,10 @@ var (
 	_ AlertProvider = (*github.AlertProvider)(nil)
 	_ AlertProvider = (*gitlab.AlertProvider)(nil)
 	_ AlertProvider = (*googlechat.AlertProvider)(nil)
+  _ AlertProvider = (*gotify.AlertProvider)(nil)
+	_ AlertProvider = (*homeassistant.AlertProvider)(nil)
+  _ AlertProvider = (*ilert.AlertProvider)(nil)
+	_ AlertProvider = (*incidentio.AlertProvider)(nil)
 	_ AlertProvider = (*jetbrainsspace.AlertProvider)(nil)
 	_ AlertProvider = (*matrix.AlertProvider)(nil)
 	_ AlertProvider = (*mattermost.AlertProvider)(nil)
@@ -94,7 +101,6 @@ var (
 	_ AlertProvider = (*telegram.AlertProvider)(nil)
 	_ AlertProvider = (*twilio.AlertProvider)(nil)
 	_ AlertProvider = (*zulip.AlertProvider)(nil)
-	_ AlertProvider = (*incidentio.AlertProvider)(nil)
 
 	// Validate config interface implementation on compile
 	_ Config[awsses.Config]         = (*awsses.Config)(nil)
@@ -105,6 +111,9 @@ var (
 	_ Config[github.Config]         = (*github.Config)(nil)
 	_ Config[gitlab.Config]         = (*gitlab.Config)(nil)
 	_ Config[googlechat.Config]     = (*googlechat.Config)(nil)
+  _ Config[gotify.Config]         = (*gotify.Config)(nil)
+	_ Config[homeassistant.Config]  = (*homeassistant.Config)(nil)
+  _ Config[ilert.Config]          = (*ilert.Config)(nil)
 	_ Config[incidentio.Config]     = (*incidentio.Config)(nil)
 	_ Config[jetbrainsspace.Config] = (*jetbrainsspace.Config)(nil)
 	_ Config[matrix.Config]         = (*matrix.Config)(nil)

alerting/provider/telegram/telegram.go

@@ -23,9 +23,10 @@ var (
 )
 
 type Config struct {
-	Token  string `yaml:"token"`
-	ID     string `yaml:"id"`
-	ApiUrl string `yaml:"api-url"`
+	Token   string `yaml:"token"`
+	ID      string `yaml:"id"`
+	TopicID string `yaml:"topic-id,omitempty"`
+	ApiUrl  string `yaml:"api-url"`
 
 	ClientConfig *client.Config `yaml:"client,omitempty"`
 }
@@ -53,6 +54,9 @@ func (cfg *Config) Merge(override *Config) {
 	if len(override.ID) > 0 {
 		cfg.ID = override.ID
 	}
+	if len(override.TopicID) > 0 {
+		cfg.TopicID = override.TopicID
+	}
 	if len(override.ApiUrl) > 0 {
 		cfg.ApiUrl = override.ApiUrl
 	}
@@ -117,6 +121,7 @@ type Body struct {
 	ChatID    string `json:"chat_id"`
 	Text      string `json:"text"`
 	ParseMode string `json:"parse_mode"`
+	TopicID   string `json:"message_thread_id,omitempty"`
 }
 
 // buildRequestBody builds the request body for the provider
@@ -150,6 +155,7 @@ func (provider *AlertProvider) buildRequestBody(cfg *Config, ep *endpoint.Endpoi
 		ChatID:    cfg.ID,
 		Text:      text,
 		ParseMode: "MARKDOWN",
+		TopicID:   cfg.TopicID,
 	})
 	return bodyAsJSON
 }

alerting/provider/telegram/telegram_test.go

@@ -154,6 +154,13 @@ func TestAlertProvider_buildRequestBody(t *testing.T) {
 			Resolved:     true,
 			ExpectedBody: "{\"chat_id\":\"123\",\"text\":\"⛑ *Gatus* \\nAn alert for *endpoint-name* has been resolved:\\n—\\n    _healthcheck passing successfully 5 time(s) in a row_\\n—   \\n*Description* \\n_description-2_  \\n\",\"parse_mode\":\"MARKDOWN\"}",
 		},
+		{
+			Name:         "send to topic",
+			Provider:     AlertProvider{DefaultConfig: Config{ID: "123", TopicID: "7"}},
+			Alert:        alert.Alert{Description: &firstDescription, SuccessThreshold: 5, FailureThreshold: 3},
+			Resolved:     false,
+			ExpectedBody: "{\"chat_id\":\"123\",\"text\":\"⛑ *Gatus* \\nAn alert for *endpoint-name* has been triggered:\\n—\\n    _healthcheck failed 3 time(s) in a row_\\n—   \\n*Description* \\n_description-1_  \\n\\n*Condition results*\\n❌ - `[CONNECTED] == true`\\n❌ - `[STATUS] == 200`\\n\",\"parse_mode\":\"MARKDOWN\",\"message_thread_id\":\"7\"}",
+		},
 	}
 	for _, scenario := range scenarios {
 		t.Run(scenario.Name, func(t *testing.T) {

api/external_endpoint.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/TwiN/gatus/v5/config"
 	"github.com/TwiN/gatus/v5/config/endpoint"
+	"github.com/TwiN/gatus/v5/metrics"
 	"github.com/TwiN/gatus/v5/storage/store"
 	"github.com/TwiN/gatus/v5/storage/store/common"
 	"github.com/TwiN/gatus/v5/watchdog"
@@ -15,6 +16,7 @@ import (
 )
 
 func CreateExternalEndpointResult(cfg *config.Config) fiber.Handler {
+	extraLabels := cfg.GetUniqueExtraMetricLabels()
 	return func(c *fiber.Ctx) error {
 		// Check if the success query parameter is present
 		success, exists := c.Queries()["success"]
@@ -46,6 +48,14 @@ func CreateExternalEndpointResult(cfg *config.Config) fiber.Handler {
 			Success:   c.QueryBool("success"),
 			Errors:    []string{},
 		}
+		if len(c.Query("duration")) > 0 {
+			parsedDuration, err := time.ParseDuration(c.Query("duration"))
+			if err != nil {
+				logr.Errorf("[api.CreateExternalEndpointResult] Invalid duration from string=%s with error: %s", c.Query("duration"), err.Error())
+				return c.Status(400).SendString("invalid duration: " + err.Error())
+			}
+			result.Duration = parsedDuration
+		}
 		if !result.Success && c.Query("error") != "" {
 			result.Errors = append(result.Errors, c.Query("error"))
 		}
@@ -64,6 +74,9 @@ func CreateExternalEndpointResult(cfg *config.Config) fiber.Handler {
 			externalEndpoint.NumberOfSuccessesInARow = convertedEndpoint.NumberOfSuccessesInARow
 			externalEndpoint.NumberOfFailuresInARow = convertedEndpoint.NumberOfFailuresInARow
 		}
+		if cfg.Metrics {
+			metrics.PublishMetricsForEndpoint(convertedEndpoint, result, extraLabels)
+		}
 		// Return the result
 		return c.Status(200).SendString("")
 	}

api/external_endpoint_test.go

@@ -70,6 +70,12 @@ func TestCreateExternalEndpointResult(t *testing.T) {
 			AuthorizationHeaderBearerToken: "Bearer token",
 			ExpectedCode:                   400,
 		},
+		{
+			Name:                           "bad-duration-value",
+			Path:                           "/api/v1/endpoints/g_n/external?success=true&duration=invalid",
+			AuthorizationHeaderBearerToken: "Bearer token",
+			ExpectedCode:                   400,
+		},
 		{
 			Name:                           "good-token-success-true",
 			Path:                           "/api/v1/endpoints/g_n/external?success=true",
@@ -82,6 +88,12 @@ func TestCreateExternalEndpointResult(t *testing.T) {
 			AuthorizationHeaderBearerToken: "Bearer token",
 			ExpectedCode:                   200,
 		},
+		{
+			Name:                           "good-duration-success-true",
+			Path:                           "/api/v1/endpoints/g_n/external?success=true&duration=10s",
+			AuthorizationHeaderBearerToken: "Bearer token",
+			ExpectedCode:                   200,
+		},
 		{
 			Name:                           "good-token-success-false",
 			Path:                           "/api/v1/endpoints/g_n/external?success=false",
@@ -118,7 +130,7 @@ func TestCreateExternalEndpointResult(t *testing.T) {
 		})
 	}
 	t.Run("verify-end-results", func(t *testing.T) {
-		endpointStatus, err := store.Get().GetEndpointStatus("g", "n", paging.NewEndpointStatusParams().WithResults(1, 10))
+		endpointStatus, err := store.Get().GetEndpointStatus("g", "n", paging.NewEndpointStatusParams().WithResults(1, 11))
 		if err != nil {
 			t.Errorf("failed to get endpoint status: %s", err.Error())
 			return
@@ -126,8 +138,8 @@ func TestCreateExternalEndpointResult(t *testing.T) {
 		if endpointStatus.Key != "g_n" {
 			t.Errorf("expected key to be g_n but got %s", endpointStatus.Key)
 		}
-		if len(endpointStatus.Results) != 5 {
-			t.Errorf("expected 3 results but got %d", len(endpointStatus.Results))
+		if len(endpointStatus.Results) != 6 {
+			t.Errorf("expected 6 results but got %d", len(endpointStatus.Results))
 		}
 		if !endpointStatus.Results[0].Success {
 			t.Errorf("expected first result to be successful")
@@ -138,8 +150,8 @@ func TestCreateExternalEndpointResult(t *testing.T) {
 		if len(endpointStatus.Results[1].Errors) > 0 {
 			t.Errorf("expected second result to have no errors")
 		}
-		if endpointStatus.Results[2].Success {
-			t.Errorf("expected third result to be unsuccessful")
+		if endpointStatus.Results[2].Duration == 0 || endpointStatus.Results[2].Duration.Seconds() != 10 {
+			t.Errorf("expected third result to have a duration of 10 seconds")
 		}
 		if endpointStatus.Results[3].Success {
 			t.Errorf("expected fourth result to be unsuccessful")
@@ -147,8 +159,11 @@ func TestCreateExternalEndpointResult(t *testing.T) {
 		if endpointStatus.Results[4].Success {
 			t.Errorf("expected fifth result to be unsuccessful")
 		}
-		if len(endpointStatus.Results[4].Errors) == 0 || endpointStatus.Results[4].Errors[0] != "failed" {
-			t.Errorf("expected fifth result to have errors: failed")
+		if endpointStatus.Results[5].Success {
+			t.Errorf("expected sixth result to be unsuccessful")
+		}
+		if len(endpointStatus.Results[5].Errors) == 0 || endpointStatus.Results[5].Errors[0] != "failed" {
+			t.Errorf("expected sixth result to have errors: failed")
 		}
 		externalEndpointFromConfig := cfg.GetExternalEndpointByKey("g_n")
 		if externalEndpointFromConfig.NumberOfFailuresInARow != 3 {

api/util.go

@@ -34,11 +34,13 @@ func extractPageAndPageSizeFromRequest(c *fiber.Ctx, maximumNumberOfResults int)
 		if err != nil {
 			pageSize = DefaultPageSize
 		}
-		if pageSize > maximumNumberOfResults {
-			pageSize = maximumNumberOfResults
-		} else if pageSize < 1 {
-			pageSize = DefaultPageSize
-		}
+	}
+	if page == 1 && pageSize > maximumNumberOfResults {
+		// If the page is 1 and the page size is greater than the maximum number of results, return
+		// no more than the maximum number of results
+		pageSize = maximumNumberOfResults
+	} else if pageSize < 1 {
+		pageSize = DefaultPageSize
 	}
 	return
 }

client/client.go

@@ -76,24 +76,37 @@ func GetDomainExpiration(hostname string) (domainExpiration time.Duration, err e
 	return domainExpiration, nil
 }
 
-// CanCreateTCPConnection checks whether a connection can be established with a TCP endpoint
-func CanCreateTCPConnection(address string, config *Config) bool {
-	conn, err := net.DialTimeout("tcp", address, config.Timeout)
-	if err != nil {
-		return false
-	}
-	_ = conn.Close()
-	return true
+// parseLocalAddressPlaceholder returns a string with the local address replaced
+func parseLocalAddressPlaceholder(item string, localAddr net.Addr) string {
+	item = strings.ReplaceAll(item, "[LOCAL_ADDRESS]", localAddr.String())
+	return item
 }
 
-// CanCreateUDPConnection checks whether a connection can be established with a UDP endpoint
-func CanCreateUDPConnection(address string, config *Config) bool {
-	conn, err := net.DialTimeout("udp", address, config.Timeout)
+// CanCreateNetworkConnection checks whether a connection can be established with a TCP or UDP endpoint
+func CanCreateNetworkConnection(netType string, address string, body string, config *Config) (bool, []byte) {
+	const (
+		MaximumMessageSize = 1024 // in bytes
+	)
+	connection, err := net.DialTimeout(netType, address, config.Timeout)
 	if err != nil {
-		return false
+		return false, nil
 	}
-	_ = conn.Close()
-	return true
+	defer connection.Close()
+	if body != "" {
+		body = parseLocalAddressPlaceholder(body, connection.LocalAddr())
+		connection.SetDeadline(time.Now().Add(config.Timeout))
+		_, err = connection.Write([]byte(body))
+		if err != nil {
+			return false, nil
+		}
+		buf := make([]byte, MaximumMessageSize)
+		n, err := connection.Read(buf)
+		if err != nil {
+			return false, nil
+		}
+		return true, buf[:n]
+	}
+	return true, nil
 }
 
 // CanCreateSCTPConnection checks whether a connection can be established with a SCTP endpoint
@@ -152,7 +165,10 @@ func CanPerformStartTLS(address string, config *Config) (connected bool, certifi
 }
 
 // CanPerformTLS checks whether a connection can be established to an address using the TLS protocol
-func CanPerformTLS(address string, config *Config) (connected bool, certificate *x509.Certificate, err error) {
+func CanPerformTLS(address string, body string, config *Config) (connected bool, response []byte, certificate *x509.Certificate, err error) {
+	const (
+		MaximumMessageSize = 1024 // in bytes
+	)
 	connection, err := tls.DialWithDialer(&net.Dialer{Timeout: config.Timeout}, "tcp", address, &tls.Config{
 		InsecureSkipVerify: config.Insecure,
 	})
@@ -166,9 +182,27 @@ func CanPerformTLS(address string, config *Config) (connected bool, certificate
 	// Reference: https://pkg.go.dev/crypto/tls#PeerCertificates
 	if len(verifiedChains) == 0 || len(verifiedChains[0]) == 0 {
 		peerCertificates := connection.ConnectionState().PeerCertificates
-		return true, peerCertificates[0], nil
+		certificate = peerCertificates[0]
+	} else {
+		certificate = verifiedChains[0][0]
 	}
-	return true, verifiedChains[0][0], nil
+	connected = true
+	if body != "" {
+		body = parseLocalAddressPlaceholder(body, connection.LocalAddr())
+		connection.SetDeadline(time.Now().Add(config.Timeout))
+		_, err = connection.Write([]byte(body))
+		if err != nil {
+			return
+		}
+		buf := make([]byte, MaximumMessageSize)
+		var n int
+		n, err = connection.Read(buf)
+		if err != nil {
+			return
+		}
+		response = buf[:n]
+	}
+	return
 }
 
 // CanCreateSSHConnection checks whether a connection can be established and a command can be executed to an address
@@ -234,6 +268,7 @@ func ExecuteSSHCommand(sshClient *ssh.Client, body string, config *Config) (bool
 	}
 	defer sshClient.Close()
 	var b Body
+	body = parseLocalAddressPlaceholder(body, sshClient.Conn.LocalAddr())
 	if err := json.Unmarshal([]byte(body), &b); err != nil {
 		return false, 0, err
 	}
@@ -286,7 +321,7 @@ func Ping(address string, config *Config) (bool, time.Duration) {
 }
 
 // QueryWebSocket opens a websocket connection, write `body` and return a message from the server
-func QueryWebSocket(address, body string, config *Config) (bool, []byte, error) {
+func QueryWebSocket(address, body string, headers map[string]string, config *Config) (bool, []byte, error) {
 	const (
 		Origin             = "http://localhost/"
 		MaximumMessageSize = 1024 // in bytes
@@ -295,8 +330,22 @@ func QueryWebSocket(address, body string, config *Config) (bool, []byte, error)
 	if err != nil {
 		return false, nil, fmt.Errorf("error configuring websocket connection: %w", err)
 	}
+	if headers != nil {
+		if wsConfig.Header == nil {
+			wsConfig.Header = make(http.Header)
+		}
+		for name, value := range headers {
+			wsConfig.Header.Set(name, value)
+		}
+	}
 	if config != nil {
 		wsConfig.Dialer = &net.Dialer{Timeout: config.Timeout}
+		wsConfig.TlsConfig = &tls.Config{
+			InsecureSkipVerify: config.Insecure,
+		}
+		if config.HasTLSConfig() && config.TLS.isValid() == nil {
+			wsConfig.TlsConfig = configureTLS(wsConfig.TlsConfig, *config.TLS)
+		}
 	}
 	// Dial URL
 	ws, err := websocket.DialConfig(wsConfig)
@@ -304,6 +353,7 @@ func QueryWebSocket(address, body string, config *Config) (bool, []byte, error)
 		return false, nil, fmt.Errorf("error dialing websocket: %w", err)
 	}
 	defer ws.Close()
+	body = parseLocalAddressPlaceholder(body, ws.LocalAddr())
 	// Write message
 	if _, err := ws.Write([]byte(body)); err != nil {
 		return false, nil, fmt.Errorf("error writing websocket body: %w", err)

client/client_test.go

@@ -223,7 +223,7 @@ func TestCanPerformTLS(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
-			connected, _, err := CanPerformTLS(tt.args.address, &Config{Insecure: tt.args.insecure, Timeout: 5 * time.Second})
+			connected, _, _, err := CanPerformTLS(tt.args.address, "", &Config{Insecure: tt.args.insecure, Timeout: 5 * time.Second})
 			if (err != nil) != tt.wantErr {
 				t.Errorf("CanPerformTLS() err=%v, wantErr=%v", err, tt.wantErr)
 				return
@@ -235,11 +235,13 @@ func TestCanPerformTLS(t *testing.T) {
 	}
 }
 
-func TestCanCreateTCPConnection(t *testing.T) {
-	if CanCreateTCPConnection("127.0.0.1", &Config{Timeout: 5 * time.Second}) {
+func TestCanCreateConnection(t *testing.T) {
+	connected, _ := CanCreateNetworkConnection("tcp", "127.0.0.1", "", &Config{Timeout: 5 * time.Second})
+	if connected {
 		t.Error("should've failed, because there's no port in the address")
 	}
-	if !CanCreateTCPConnection("1.1.1.1:53", &Config{Timeout: 5 * time.Second}) {
+	connected, _ = CanCreateNetworkConnection("tcp", "1.1.1.1:53", "", &Config{Timeout: 5 * time.Second})
+	if !connected {
 		t.Error("should've succeeded, because that IP should always™ be up")
 	}
 }
@@ -303,11 +305,11 @@ func TestHttpClientProvidesOAuth2BearerToken(t *testing.T) {
 }
 
 func TestQueryWebSocket(t *testing.T) {
-	_, _, err := QueryWebSocket("", "body", &Config{Timeout: 2 * time.Second})
+	_, _, err := QueryWebSocket("", "body", nil, &Config{Timeout: 2 * time.Second})
 	if err == nil {
 		t.Error("expected an error due to the address being invalid")
 	}
-	_, _, err = QueryWebSocket("ws://example.org", "body", &Config{Timeout: 2 * time.Second})
+	_, _, err = QueryWebSocket("ws://example.org", "body", nil, &Config{Timeout: 2 * time.Second})
 	if err == nil {
 		t.Error("expected an error due to the target not being websocket-friendly")
 	}

config/config.go

@@ -22,7 +22,6 @@ import (
 	"github.com/TwiN/gatus/v5/security"
 	"github.com/TwiN/gatus/v5/storage"
 	"github.com/TwiN/logr"
-	"github.com/gofiber/fiber/v2/log"
 	"gopkg.in/yaml.v3"
 )
 
@@ -103,10 +102,29 @@ type Config struct {
 	lastFileModTime time.Time // last modification time
 }
 
+// GetUniqueExtraMetricLabels returns a slice of unique metric labels from all enabled endpoints
+// in the configuration. It iterates through each endpoint, checks if it is enabled,
+// and then collects unique labels from the endpoint's labels map.
+func (config *Config) GetUniqueExtraMetricLabels() []string {
+	labels := make([]string, 0)
+	for _, ep := range config.Endpoints {
+		if !ep.IsEnabled() {
+			continue
+		}
+		for label := range ep.ExtraLabels {
+			if contains(labels, label) {
+				continue
+			}
+			labels = append(labels, label)
+		}
+	}
+	return labels
+}
+
 func (config *Config) GetEndpointByKey(key string) *endpoint.Endpoint {
 	for i := 0; i < len(config.Endpoints); i++ {
 		ep := config.Endpoints[i]
-		if ep.Key() == key {
+		if ep.Key() == strings.ToLower(key) {
 			return ep
 		}
 	}
@@ -116,7 +134,7 @@ func (config *Config) GetEndpointByKey(key string) *endpoint.Endpoint {
 func (config *Config) GetExternalEndpointByKey(key string) *endpoint.ExternalEndpoint {
 	for i := 0; i < len(config.ExternalEndpoints); i++ {
 		ee := config.ExternalEndpoints[i]
-		if ee.Key() == key {
+		if ee.Key() == strings.ToLower(key) {
 			return ee
 		}
 	}
@@ -411,6 +429,9 @@ func validateAlertingConfig(alertingConfig *alerting.Config, endpoints []*endpoi
 		alert.TypeGitea,
 		alert.TypeGoogleChat,
 		alert.TypeGotify,
+		alert.TypeHomeAssistant,
+		alert.TypeIlert,
+		alert.TypeIncidentIO,
 		alert.TypeJetBrainsSpace,
 		alert.TypeMatrix,
 		alert.TypeMattermost,
@@ -425,7 +446,6 @@ func validateAlertingConfig(alertingConfig *alerting.Config, endpoints []*endpoi
 		alert.TypeTelegram,
 		alert.TypeTwilio,
 		alert.TypeZulip,
-		alert.TypeIncidentIO,
 	}
 	var validProviders, invalidProviders []alert.Type
 	for _, alertType := range alertTypes {
@@ -442,7 +462,7 @@ func validateAlertingConfig(alertingConfig *alerting.Config, endpoints []*endpoi
 								// Validate the endpoint alert's overrides, if applicable
 								if len(endpointAlert.ProviderOverride) > 0 {
 									if err = alertProvider.ValidateOverrides(ep.Group, endpointAlert); err != nil {
-										log.Warnf("[config.validateAlertingConfig] endpoint with key=%s has invalid overrides for provider=%s: %s", ep.Key(), alertType, err.Error())
+										logr.Warnf("[config.validateAlertingConfig] endpoint with key=%s has invalid overrides for provider=%s: %s", ep.Key(), alertType, err.Error())
 									}
 								}
 							}
@@ -456,7 +476,7 @@ func validateAlertingConfig(alertingConfig *alerting.Config, endpoints []*endpoi
 								// Validate the endpoint alert's overrides, if applicable
 								if len(endpointAlert.ProviderOverride) > 0 {
 									if err = alertProvider.ValidateOverrides(ee.Group, endpointAlert); err != nil {
-										log.Warnf("[config.validateAlertingConfig] endpoint with key=%s has invalid overrides for provider=%s: %s", ee.Key(), alertType, err.Error())
+										logr.Warnf("[config.validateAlertingConfig] endpoint with key=%s has invalid overrides for provider=%s: %s", ee.Key(), alertType, err.Error())
 									}
 								}
 							}

config/config_test.go

@@ -124,7 +124,7 @@ endpoints:
 			name:       "dir-with-two-config-files",
 			configPath: dir,
 			pathAndFiles: map[string]string{
-				"config.yaml": `endpoints: 
+				"config.yaml": `endpoints:
   - name: one
     url: https://example.com
     conditions:
@@ -135,7 +135,7 @@ endpoints:
     url: https://example.org
     conditions:
       - "len([BODY]) > 0"`,
-				"config.yml": `endpoints: 
+				"config.yml": `endpoints:
   - name: three
     url: https://twin.sh/health
     conditions:
@@ -237,7 +237,7 @@ endpoints:
 	for _, scenario := range scenarios {
 		t.Run(scenario.name, func(t *testing.T) {
 			for path, content := range scenario.pathAndFiles {
-				if err := os.WriteFile(filepath.Join(dir, path), []byte(content), 0644); err != nil {
+				if err := os.WriteFile(filepath.Join(dir, path), []byte(content), 0o644); err != nil {
 					t.Fatalf("[%s] failed to write file: %v", scenario.name, err)
 				}
 			}
@@ -282,7 +282,7 @@ func TestConfig_HasLoadedConfigurationBeenModified(t *testing.T) {
     url: https://twin.sh/health
     conditions:
       - "[STATUS] == 200"
-`), 0644)
+`), 0o644)
 
 	t.Run("config-file-as-config-path", func(t *testing.T) {
 		config, err := LoadConfiguration(configFilePath)
@@ -298,7 +298,7 @@ func TestConfig_HasLoadedConfigurationBeenModified(t *testing.T) {
   - name: website
     url: https://twin.sh/health
     conditions:
-      - "[STATUS] == 200"`), 0644); err != nil {
+      - "[STATUS] == 200"`), 0o644); err != nil {
 			t.Fatalf("failed to overwrite config file: %v", err)
 		}
 		if !config.HasLoadedConfigurationBeenModified() {
@@ -315,7 +315,7 @@ func TestConfig_HasLoadedConfigurationBeenModified(t *testing.T) {
 		}
 		time.Sleep(time.Second) // Because the file mod time only has second precision, we have to wait for a second
 		// Update the config file
-		if err = os.WriteFile(filepath.Join(dir, "metrics.yaml"), []byte(`metrics: true`), 0644); err != nil {
+		if err = os.WriteFile(filepath.Join(dir, "metrics.yaml"), []byte(`metrics: true`), 0o644); err != nil {
 			t.Fatalf("failed to overwrite config file: %v", err)
 		}
 		if !config.HasLoadedConfigurationBeenModified() {
@@ -713,7 +713,7 @@ func TestParseAndValidateBadConfigBytes(t *testing.T) {
 	_, err := parseAndValidateConfigBytes([]byte(`
 badconfig:
   - asdsa: w0w
-    usadasdrl: asdxzczxc	
+    usadasdrl: asdxzczxc
     asdas:
       - soup
 `))
@@ -1943,3 +1943,114 @@ func TestGetAlertingProviderByAlertType(t *testing.T) {
 		})
 	}
 }
+
+func TestConfig_GetUniqueExtraMetricLabels(t *testing.T) {
+	tests := []struct {
+		name     string
+		config   *Config
+		expected []string
+	}{
+		{
+			name: "no-endpoints",
+			config: &Config{
+				Endpoints: []*endpoint.Endpoint{},
+			},
+			expected: []string{},
+		},
+		{
+			name: "single-endpoint-no-labels",
+			config: &Config{
+				Endpoints: []*endpoint.Endpoint{
+					{
+						Name: "endpoint1",
+						URL:  "https://example.com",
+					},
+				},
+			},
+			expected: []string{},
+		},
+		{
+			name: "single-endpoint-with-labels",
+			config: &Config{
+				Endpoints: []*endpoint.Endpoint{
+					{
+						Name:    "endpoint1",
+						URL:     "https://example.com",
+						Enabled: toPtr(true),
+						ExtraLabels: map[string]string{
+							"env":  "production",
+							"team": "backend",
+						},
+					},
+				},
+			},
+			expected: []string{"env", "team"},
+		},
+		{
+			name: "multiple-endpoints-with-labels",
+			config: &Config{
+				Endpoints: []*endpoint.Endpoint{
+					{
+						Name:    "endpoint1",
+						URL:     "https://example.com",
+						Enabled: toPtr(true),
+						ExtraLabels: map[string]string{
+							"env":    "production",
+							"team":   "backend",
+							"module": "auth",
+						},
+					},
+					{
+						Name:    "endpoint2",
+						URL:     "https://example.org",
+						Enabled: toPtr(true),
+						ExtraLabels: map[string]string{
+							"env":  "staging",
+							"team": "frontend",
+						},
+					},
+				},
+			},
+			expected: []string{"env", "team", "module"},
+		},
+		{
+			name: "multiple-endpoints-with-some-disabled",
+			config: &Config{
+				Endpoints: []*endpoint.Endpoint{
+					{
+						Name:    "endpoint1",
+						URL:     "https://example.com",
+						Enabled: toPtr(true),
+						ExtraLabels: map[string]string{
+							"env":  "production",
+							"team": "backend",
+						},
+					},
+					{
+						Name:    "endpoint2",
+						URL:     "https://example.org",
+						Enabled: toPtr(false),
+						ExtraLabels: map[string]string{
+							"module": "auth",
+						},
+					},
+				},
+			},
+			expected: []string{"env", "team"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			labels := tt.config.GetUniqueExtraMetricLabels()
+			if len(labels) != len(tt.expected) {
+				t.Errorf("expected %d labels, got %d", len(tt.expected), len(labels))
+			}
+			for _, label := range tt.expected {
+				if !contains(labels, label) {
+					t.Errorf("expected label %s to be present", label)
+				}
+			}
+		})
+	}
+}

config/connectivity/connectivity.go

@@ -42,7 +42,8 @@ type Checker struct {
 }
 
 func (c *Checker) Check() bool {
-	return client.CanCreateTCPConnection(c.Target, &client.Config{Timeout: 5 * time.Second})
+	connected, _ := client.CanCreateNetworkConnection("tcp", c.Target, "", &client.Config{Timeout: 5 * time.Second})
+	return connected
 }
 
 func (c *Checker) IsConnected() bool {

config/endpoint/endpoint.go

@@ -7,9 +7,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math/rand"
 	"net"
 	"net/http"
 	"net/url"
+	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
@@ -96,6 +99,9 @@ type Endpoint struct {
 	// Headers of the request
 	Headers map[string]string `yaml:"headers,omitempty"`
 
+	// ExtraLabels are key-value pairs that can be used to metric the endpoint
+	ExtraLabels map[string]string `yaml:"extra-labels,omitempty"`
+
 	// Interval is the duration to wait between every status check
 	Interval time.Duration `yaml:"interval,omitempty"`
 
@@ -125,6 +131,9 @@ type Endpoint struct {
 
 	// NumberOfSuccessesInARow is the number of successful evaluations in a row
 	NumberOfSuccessesInARow int `yaml:"-"`
+
+	// LastReminderSent is the time at which the last reminder was sent for this endpoint.
+	LastReminderSent time.Time `yaml:"-"`
 }
 
 // IsEnabled returns whether the endpoint is enabled or not
@@ -229,7 +238,7 @@ func (e *Endpoint) ValidateAndSetDefaults() error {
 		}
 	}
 	// Make sure that the request can be created
-	_, err := http.NewRequest(e.Method, e.URL, bytes.NewBuffer([]byte(e.Body)))
+	_, err := http.NewRequest(e.Method, e.URL, bytes.NewBuffer([]byte(e.getParsedBody())))
 	if err != nil {
 		return err
 	}
@@ -326,6 +335,29 @@ func (e *Endpoint) EvaluateHealth() *Result {
 	return result
 }
 
+func (e *Endpoint) getParsedBody() string {
+	body := e.Body
+	body = strings.ReplaceAll(body, "[ENDPOINT_NAME]", e.Name)
+	body = strings.ReplaceAll(body, "[ENDPOINT_GROUP]", e.Group)
+	body = strings.ReplaceAll(body, "[ENDPOINT_URL]", e.URL)
+	randRegex, err := regexp.Compile(`\[RANDOM_STRING_\d+\]`)
+	if err == nil {
+		body = randRegex.ReplaceAllStringFunc(body, func(match string) string {
+			n, _ := strconv.Atoi(match[15 : len(match)-1])
+			if n > 8192 {
+				n = 8192 // Limit the length of the random string to 8192 bytes to avoid excessive memory usage
+			}
+			const availableCharacterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+			b := make([]byte, n)
+			for i := range b {
+				b[i] = availableCharacterBytes[rand.Intn(len(availableCharacterBytes))]
+			}
+			return string(b)
+		})
+	}
+	return body
+}
+
 func (e *Endpoint) getIP(result *Result) {
 	if ips, err := net.LookupIP(result.Hostname); err != nil {
 		result.AddError(err.Error())
@@ -356,7 +388,7 @@ func (e *Endpoint) call(result *Result) {
 		if endpointType == TypeSTARTTLS {
 			result.Connected, certificate, err = client.CanPerformStartTLS(strings.TrimPrefix(e.URL, "starttls://"), e.ClientConfig)
 		} else {
-			result.Connected, certificate, err = client.CanPerformTLS(strings.TrimPrefix(e.URL, "tls://"), e.ClientConfig)
+			result.Connected, result.Body, certificate, err = client.CanPerformTLS(strings.TrimPrefix(e.URL, "tls://"), e.getParsedBody(), e.ClientConfig)
 		}
 		if err != nil {
 			result.AddError(err.Error())
@@ -365,10 +397,10 @@ func (e *Endpoint) call(result *Result) {
 		result.Duration = time.Since(startTime)
 		result.CertificateExpiration = time.Until(certificate.NotAfter)
 	} else if endpointType == TypeTCP {
-		result.Connected = client.CanCreateTCPConnection(strings.TrimPrefix(e.URL, "tcp://"), e.ClientConfig)
+		result.Connected, result.Body = client.CanCreateNetworkConnection("tcp", strings.TrimPrefix(e.URL, "tcp://"), e.getParsedBody(), e.ClientConfig)
 		result.Duration = time.Since(startTime)
 	} else if endpointType == TypeUDP {
-		result.Connected = client.CanCreateUDPConnection(strings.TrimPrefix(e.URL, "udp://"), e.ClientConfig)
+		result.Connected, result.Body = client.CanCreateNetworkConnection("udp", strings.TrimPrefix(e.URL, "udp://"), e.getParsedBody(), e.ClientConfig)
 		result.Duration = time.Since(startTime)
 	} else if endpointType == TypeSCTP {
 		result.Connected = client.CanCreateSCTPConnection(strings.TrimPrefix(e.URL, "sctp://"), e.ClientConfig)
@@ -376,7 +408,16 @@ func (e *Endpoint) call(result *Result) {
 	} else if endpointType == TypeICMP {
 		result.Connected, result.Duration = client.Ping(strings.TrimPrefix(e.URL, "icmp://"), e.ClientConfig)
 	} else if endpointType == TypeWS {
-		result.Connected, result.Body, err = client.QueryWebSocket(e.URL, e.Body, e.ClientConfig)
+		wsHeaders := map[string]string{}
+		if e.Headers != nil {
+			for k, v := range e.Headers {
+				wsHeaders[k] = v
+			}
+		}
+		if _, exists := wsHeaders["User-Agent"]; !exists {
+			wsHeaders["User-Agent"] = GatusUserAgent
+		}
+		result.Connected, result.Body, err = client.QueryWebSocket(e.URL, e.getParsedBody(), wsHeaders, e.ClientConfig)
 		if err != nil {
 			result.AddError(err.Error())
 			return
@@ -385,8 +426,7 @@ func (e *Endpoint) call(result *Result) {
 	} else if endpointType == TypeSSH {
 		// If there's no username/password specified, attempt to validate just the SSH banner
 		if len(e.SSHConfig.Username) == 0 && len(e.SSHConfig.Password) == 0 {
-			result.Connected, result.HTTPStatus, err =
-				client.CheckSSHBanner(strings.TrimPrefix(e.URL, "ssh://"), e.ClientConfig)
+			result.Connected, result.HTTPStatus, err = client.CheckSSHBanner(strings.TrimPrefix(e.URL, "ssh://"), e.ClientConfig)
 			if err != nil {
 				result.AddError(err.Error())
 				return
@@ -401,7 +441,7 @@ func (e *Endpoint) call(result *Result) {
 			result.AddError(err.Error())
 			return
 		}
-		result.Success, result.HTTPStatus, err = client.ExecuteSSHCommand(cli, e.Body, e.ClientConfig)
+		result.Success, result.HTTPStatus, err = client.ExecuteSSHCommand(cli, e.getParsedBody(), e.ClientConfig)
 		if err != nil {
 			result.AddError(err.Error())
 			return
@@ -435,12 +475,12 @@ func (e *Endpoint) buildHTTPRequest() *http.Request {
 	var bodyBuffer *bytes.Buffer
 	if e.GraphQL {
 		graphQlBody := map[string]string{
-			"query": e.Body,
+			"query": e.getParsedBody(),
 		}
 		body, _ := json.Marshal(graphQlBody)
 		bodyBuffer = bytes.NewBuffer(body)
 	} else {
-		bodyBuffer = bytes.NewBuffer([]byte(e.Body))
+		bodyBuffer = bytes.NewBuffer([]byte(e.getParsedBody()))
 	}
 	request, _ := http.NewRequest(e.Method, e.URL, bodyBuffer)
 	for k, v := range e.Headers {

config/endpoint/external_endpoint.go

@@ -2,13 +2,19 @@ package endpoint
 
 import (
 	"errors"
+	"time"
 
 	"github.com/TwiN/gatus/v5/alerting/alert"
+	"github.com/TwiN/gatus/v5/config/endpoint/heartbeat"
+	"github.com/TwiN/gatus/v5/config/maintenance"
 )
 
 var (
 	// ErrExternalEndpointWithNoToken is the error with which Gatus will panic if an external endpoint is configured without a token.
 	ErrExternalEndpointWithNoToken = errors.New("you must specify a token for each external endpoint")
+
+	// ErrExternalEndpointHeartbeatIntervalTooLow is the error with which Gatus will panic if an external endpoint's heartbeat interval is less than 10 seconds.
+	ErrExternalEndpointHeartbeatIntervalTooLow = errors.New("heartbeat interval must be at least 10 seconds")
 )
 
 // ExternalEndpoint is an endpoint whose result is pushed from outside Gatus, which means that
@@ -30,6 +36,12 @@ type ExternalEndpoint struct {
 	// Alerts is the alerting configuration for the endpoint in case of failure
 	Alerts []*alert.Alert `yaml:"alerts,omitempty"`
 
+	// MaintenanceWindow is the configuration for per-endpoint maintenance windows
+	MaintenanceWindows []*maintenance.Config `yaml:"maintenance-windows,omitempty"`
+
+	// Heartbeat is the configuration that checks if the external endpoint has received new results when it should have.
+	Heartbeat heartbeat.Config `yaml:"heartbeat,omitempty"`
+
 	// NumberOfFailuresInARow is the number of unsuccessful evaluations in a row
 	NumberOfFailuresInARow int `yaml:"-"`
 
@@ -45,6 +57,10 @@ func (externalEndpoint *ExternalEndpoint) ValidateAndSetDefaults() error {
 	if len(externalEndpoint.Token) == 0 {
 		return ErrExternalEndpointWithNoToken
 	}
+	if externalEndpoint.Heartbeat.Interval != 0 && externalEndpoint.Heartbeat.Interval < 10*time.Second {
+		// If the heartbeat interval is set (non-0), it must be at least 10 seconds.
+		return ErrExternalEndpointHeartbeatIntervalTooLow
+	}
 	return nil
 }
 

config/endpoint/heartbeat/heartbeat.go

@@ -0,0 +1,11 @@
+package heartbeat
+
+import "time"
+
+// Config used to check if the external endpoint has received new results when it should have.
+// This configuration is used to trigger alerts when an external endpoint has no new results for a defined period of time
+type Config struct {
+	// Interval is the time interval at which Gatus verifies whether the external endpoint has received new results
+	// If no new result is received within the interval, the endpoint is marked as failed and alerts are triggered
+	Interval time.Duration `yaml:"interval"`
+}

config/ui/ui.go

@@ -35,7 +35,10 @@ type Config struct {
 	CustomCSS   string   `yaml:"custom-css,omitempty"`  // Custom CSS to include in the page
 	DarkMode    *bool    `yaml:"dark-mode,omitempty"`   // DarkMode is a flag to enable dark mode by default
 
-	MaximumNumberOfResults int // MaximumNumberOfResults to display on the page, it's not configurable because we're passing it from the storage config
+	//////////////////////////////////////////////
+	// Non-configurable - used for UI rendering //
+	//////////////////////////////////////////////
+	MaximumNumberOfResults int `yaml:"-"` // MaximumNumberOfResults to display on the page, it's not configurable because we're passing it from the storage config
 }
 
 func (cfg *Config) IsDarkMode() bool {

config/util.go

@@ -0,0 +1,16 @@
+package config
+
+// toPtr returns a pointer to the given value
+func toPtr[T any](value T) *T {
+	return &value
+}
+
+// contains checks if a key exists in the slice
+func contains[T comparable](slice []T, key T) bool {
+	for _, item := range slice {
+		if item == key {
+			return true
+		}
+	}
+	return false
+}

go.mod

@@ -3,39 +3,40 @@ module github.com/TwiN/gatus/v5
 go 1.24.1
 
 require (
-	code.gitea.io/sdk/gitea v0.19.0
+	code.gitea.io/sdk/gitea v0.21.0
 	github.com/TwiN/deepmerge v0.2.2
 	github.com/TwiN/g8/v2 v2.0.0
 	github.com/TwiN/gocache/v2 v2.2.2
 	github.com/TwiN/health v1.6.0
 	github.com/TwiN/logr v0.3.1
-	github.com/TwiN/whois v1.1.10
-	github.com/aws/aws-sdk-go v1.55.6
+	github.com/TwiN/whois v1.1.11
+	github.com/aws/aws-sdk-go v1.55.8
 	github.com/coreos/go-oidc/v3 v3.14.1
-	github.com/gofiber/fiber/v2 v2.52.6
+	github.com/gofiber/fiber/v2 v2.52.8
 	github.com/google/go-github/v48 v48.2.0
 	github.com/google/uuid v1.6.0
 	github.com/ishidawataru/sctp v0.0.0-20230406120618-7ff4192f6ff2
 	github.com/lib/pq v1.10.9
-	github.com/miekg/dns v1.1.65
+	github.com/miekg/dns v1.1.67
 	github.com/prometheus-community/pro-bing v0.6.1
-	github.com/prometheus/client_golang v1.21.1
-	github.com/valyala/fasthttp v1.60.0
+	github.com/prometheus/client_golang v1.22.0
+	github.com/valyala/fasthttp v1.64.0
 	github.com/wcharczuk/go-chart/v2 v2.1.2
-	golang.org/x/crypto v0.38.0
-	golang.org/x/net v0.40.0
-	golang.org/x/oauth2 v0.28.0
-	google.golang.org/api v0.228.0
+	golang.org/x/crypto v0.40.0
+	golang.org/x/net v0.42.0
+	golang.org/x/oauth2 v0.30.0
+	google.golang.org/api v0.242.0
 	gopkg.in/mail.v2 v2.3.1
 	gopkg.in/yaml.v3 v3.0.1
-	modernc.org/sqlite v1.37.0
+	modernc.org/sqlite v1.38.2
 )
 
 require (
-	cloud.google.com/go/auth v0.15.0 // indirect
+	cloud.google.com/go/auth v0.16.2 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
-	cloud.google.com/go/compute/metadata v0.6.0 // indirect
-	github.com/andybalholm/brotli v1.1.1 // indirect
+	cloud.google.com/go/compute/metadata v0.7.0 // indirect
+	github.com/42wim/httpsig v1.2.2 // indirect
+	github.com/andybalholm/brotli v1.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davidmz/go-pageant v1.0.2 // indirect
@@ -49,8 +50,8 @@ require (
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
-	github.com/googleapis/gax-go/v2 v2.14.1 // indirect
-	github.com/hashicorp/go-version v1.6.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.14.2 // indirect
+	github.com/hashicorp/go-version v1.7.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
@@ -66,22 +67,22 @@ require (
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect
-	go.opentelemetry.io/otel v1.34.0 // indirect
-	go.opentelemetry.io/otel/metric v1.34.0 // indirect
-	go.opentelemetry.io/otel/trace v1.34.0 // indirect
-	golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
+	go.opentelemetry.io/otel v1.36.0 // indirect
+	go.opentelemetry.io/otel/metric v1.36.0 // indirect
+	go.opentelemetry.io/otel/trace v1.36.0 // indirect
+	golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect
 	golang.org/x/image v0.18.0 // indirect
-	golang.org/x/mod v0.24.0 // indirect
-	golang.org/x/sync v0.14.0 // indirect
-	golang.org/x/sys v0.33.0 // indirect
-	golang.org/x/text v0.25.0 // indirect
-	golang.org/x/tools v0.31.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250313205543-e70fdf4c4cb4 // indirect
-	google.golang.org/grpc v1.71.0 // indirect
+	golang.org/x/mod v0.25.0 // indirect
+	golang.org/x/sync v0.16.0 // indirect
+	golang.org/x/sys v0.34.0 // indirect
+	golang.org/x/text v0.27.0 // indirect
+	golang.org/x/tools v0.34.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
+	google.golang.org/grpc v1.73.0 // indirect
 	google.golang.org/protobuf v1.36.6 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
-	modernc.org/libc v1.62.1 // indirect
+	modernc.org/libc v1.66.3 // indirect
 	modernc.org/mathutil v1.7.1 // indirect
-	modernc.org/memory v1.9.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
 )

go.sum

@@ -1,11 +1,13 @@
-cloud.google.com/go/auth v0.15.0 h1:Ly0u4aA5vG/fsSsxu98qCQBemXtAtJf+95z9HK+cxps=
-cloud.google.com/go/auth v0.15.0/go.mod h1:WJDGqZ1o9E9wKIL+IwStfyn/+s59zl4Bi+1KQNVXLZ8=
+cloud.google.com/go/auth v0.16.2 h1:QvBAGFPLrDeoiNjyfVunhQ10HKNYuOwZ5noee0M5df4=
+cloud.google.com/go/auth v0.16.2/go.mod h1:sRBas2Y1fB1vZTdurouM0AzuYQBMZinrUYL8EufhtEA=
 cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
 cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
-cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I=
-cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg=
-code.gitea.io/sdk/gitea v0.19.0 h1:8I6s1s4RHgzxiPHhOQdgim1RWIRcr0LVMbHBjBFXq4Y=
-code.gitea.io/sdk/gitea v0.19.0/go.mod h1:IG9xZJoltDNeDSW0qiF2Vqx5orMWa7OhVWrjvrd5NpI=
+cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU=
+cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo=
+code.gitea.io/sdk/gitea v0.21.0 h1:69n6oz6kEVHRo1+APQQyizkhrZrLsTLXey9142pfkD4=
+code.gitea.io/sdk/gitea v0.21.0/go.mod h1:tnBjVhuKJCn8ibdyyhvUyxrR1Ca2KHEoTWoukNhXQPA=
+github.com/42wim/httpsig v1.2.2 h1:ofAYoHUNs/MJOLqQ8hIxeyz2QxOz8qdSVvp3PX/oPgA=
+github.com/42wim/httpsig v1.2.2/go.mod h1:P/UYo7ytNBFwc+dg35IubuAUIs8zj5zzFIgUCEl55WY=
 github.com/TwiN/deepmerge v0.2.2 h1:FUG9QMIYg/j2aQyPPhA3XTFJwXSNHI/swaR4Lbyxwg4=
 github.com/TwiN/deepmerge v0.2.2/go.mod h1:4OHvjV3pPNJCJZBHswYAwk6rxiD8h8YZ+9cPo7nu4oI=
 github.com/TwiN/g8/v2 v2.0.0 h1:+hwIbRLMhDd2iwHzkZUPp2FkX7yTx8ddYOnS91HkDqQ=
@@ -16,12 +18,12 @@ github.com/TwiN/health v1.6.0 h1:L2ks575JhRgQqWWOfKjw9B0ec172hx7GdToqkYUycQM=
 github.com/TwiN/health v1.6.0/go.mod h1:Z6TszwQPMvtSiVx1QMidVRgvVr4KZGfiwqcD7/Z+3iw=
 github.com/TwiN/logr v0.3.1 h1:CfTKA83jUmsAoxqrr3p4JxEkqXOBnEE9/f35L5MODy4=
 github.com/TwiN/logr v0.3.1/go.mod h1:BZgZFYq6fQdU3KtR8qYato3zUEw53yQDaIuujHb55Jw=
-github.com/TwiN/whois v1.1.10 h1:OdnxMRPlegKr+ypwMKq5VpJ8QoD6F2e5gY+MKTs9VyA=
-github.com/TwiN/whois v1.1.10/go.mod h1:TjipCMpJRAJYKmtz/rXQBU6UGxMh6bk8SHazu7OMnQE=
-github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
-github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
-github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk=
-github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
+github.com/TwiN/whois v1.1.11 h1:lYiYgPRSQ3kH8sQfgHcBY/uNSGGvWPRikEjn+LJZ9+Q=
+github.com/TwiN/whois v1.1.11/go.mod h1:TjipCMpJRAJYKmtz/rXQBU6UGxMh6bk8SHazu7OMnQE=
+github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
+github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
+github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
+github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -46,8 +48,8 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/gofiber/fiber/v2 v2.52.6 h1:Rfp+ILPiYSvvVuIPvxrBns+HJp8qGLDnLJawAu27XVI=
-github.com/gofiber/fiber/v2 v2.52.6/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
+github.com/gofiber/fiber/v2 v2.52.8 h1:xl4jJQ0BV5EJTA2aWiKw/VddRpHrKeZLF0QPUxqn0x4=
+github.com/gofiber/fiber/v2 v2.52.8/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
@@ -68,10 +70,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4=
 github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
-github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q=
-github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA=
-github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
-github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/googleapis/gax-go/v2 v2.14.2 h1:eBLnkZ9635krYIPD+ag1USrOAI0Nr0QYF3+/3GqO0k0=
+github.com/googleapis/gax-go/v2 v2.14.2/go.mod h1:ON64QhlJkhVtSqp4v1uaK92VyZ2gmvDQsweuyLV+8+w=
+github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
+github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/ishidawataru/sctp v0.0.0-20230406120618-7ff4192f6ff2 h1:i2fYnDurfLlJH8AyyMOnkLHnHeP8Ff/DDpuZA/D3bPo=
 github.com/ishidawataru/sctp v0.0.0-20230406120618-7ff4192f6ff2/go.mod h1:co9pwDoBCm1kGxawmb4sPq0cSIOOWNPT4KnHotMP1Zg=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
@@ -95,8 +97,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/miekg/dns v1.1.65 h1:0+tIPHzUW0GCge7IiK3guGP57VAw7hoPDfApjkMD1Fc=
-github.com/miekg/dns v1.1.65/go.mod h1:Dzw9769uoKVaLuODMDZz9M6ynFU6Em65csPuoi8G0ck=
+github.com/miekg/dns v1.1.67 h1:kg0EHj0G4bfT5/oOys6HhZw4vmMlnoZ+gDu8tJ/AlI0=
+github.com/miekg/dns v1.1.67/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
@@ -105,8 +107,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus-community/pro-bing v0.6.1 h1:EQukUOma9YFZRPe4DGSscxUf9LH07rpqwisNWjSZrgU=
 github.com/prometheus-community/pro-bing v0.6.1/go.mod h1:jNCOI3D7pmTCeaoF41cNS6uaxeFY/Gmc3ffwbuJVzAQ=
-github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk=
-github.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg=
+github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
+github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
 github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
 github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
 github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
@@ -125,8 +127,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.60.0 h1:kBRYS0lOhVJ6V+bYN8PqAHELKHtXqwq9zNMLKx1MBsw=
-github.com/valyala/fasthttp v1.60.0/go.mod h1:iY4kDgV3Gc6EqhRZ8icqcmlG6bqhcDXfuHgTO4FXCvc=
+github.com/valyala/fasthttp v1.64.0 h1:QBygLLQmiAyiXuRhthf0tuRkqAFcrC42dckN2S+N3og=
+github.com/valyala/fasthttp v1.64.0/go.mod h1:dGmFxwkWXSK0NbOSJuF7AMVzU+lkHz0wQVvVITv2UQA=
 github.com/wcharczuk/go-chart/v2 v2.1.2 h1:Y17/oYNuXwZg6TFag06qe8sBajwwsuvPiJJXcUcLL6E=
 github.com/wcharczuk/go-chart/v2 v2.1.2/go.mod h1:Zi4hbaqlWpYajnXB2K22IUYVXRXaLfSGNNR7P4ukyyQ=
 github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
@@ -134,20 +136,20 @@ github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3i
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
 go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 h1:rgMkmiGfix9vFJDcDi1PK8WEQP4FLQwLDfhp5ZLpFeE=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0/go.mod h1:ijPqXp5P6IRRByFVVg9DY8P5HkxkHE5ARIa+86aXPf4=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSGJt/Ao6Gp4CXckLxVRRsRgDHoI8XjbL3PDl8s=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I=
-go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY=
-go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI=
-go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ=
-go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE=
-go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A=
-go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU=
-go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk=
-go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w=
-go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k=
-go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
+go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg=
+go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E=
+go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE=
+go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs=
+go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs=
+go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY=
+go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis=
+go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4=
+go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w=
+go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
@@ -155,10 +157,10 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
-golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
-golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
-golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw=
-golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM=
+golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM=
+golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY=
+golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o=
+golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
 golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ=
 golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -166,8 +168,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
-golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
+golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w=
+golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
@@ -177,18 +179,18 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
-golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
-golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
-golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc=
-golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
+golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs=
+golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8=
+golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
+golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
-golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -202,8 +204,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
-golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
+golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -212,8 +214,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
-golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
-golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
+golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg=
+golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@@ -223,26 +225,26 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
-golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
-golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
-golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
-golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
+golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
+golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=
+golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
+golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU=
-golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ=
+golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo=
+golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.228.0 h1:X2DJ/uoWGnY5obVjewbp8icSL5U4FzuCfy9OjbLSnLs=
-google.golang.org/api v0.228.0/go.mod h1:wNvRS1Pbe8r4+IfBIniV8fwCpGwTrYa+kMUDiC5z5a4=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250313205543-e70fdf4c4cb4 h1:iK2jbkWL86DXjEx0qiHcRE9dE4/Ahua5k6V8OWFb//c=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250313205543-e70fdf4c4cb4/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I=
-google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg=
-google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec=
+google.golang.org/api v0.242.0 h1:7Lnb1nfnpvbkCiZek6IXKdJ0MFuAZNAJKQfA1ws62xg=
+google.golang.org/api v0.242.0/go.mod h1:cOVEm2TpdAGHL2z+UwyS+kmlGr3bVWQQ6sYEqkKje50=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
+google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok=
+google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc=
 google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
 google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
@@ -257,26 +259,28 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-modernc.org/cc/v4 v4.25.2 h1:T2oH7sZdGvTaie0BRNFbIYsabzCxUQg8nLqCdQ2i0ic=
-modernc.org/cc/v4 v4.25.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
-modernc.org/ccgo/v4 v4.25.1 h1:TFSzPrAGmDsdnhT9X2UrcPMI3N/mJ9/X9ykKXwLhDsU=
-modernc.org/ccgo/v4 v4.25.1/go.mod h1:njjuAYiPflywOOrm3B7kCB444ONP5pAVr8PIEoE0uDw=
-modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
-modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
+modernc.org/cc/v4 v4.26.2 h1:991HMkLjJzYBIfha6ECZdjrIYz2/1ayr+FL8GN+CNzM=
+modernc.org/cc/v4 v4.26.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
+modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU=
+modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE=
+modernc.org/fileutil v1.3.8 h1:qtzNm7ED75pd1C7WgAGcK4edm4fvhtBsEiI/0NQ54YM=
+modernc.org/fileutil v1.3.8/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
 modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
 modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
-modernc.org/libc v1.62.1 h1:s0+fv5E3FymN8eJVmnk0llBe6rOxCu/DEU+XygRbS8s=
-modernc.org/libc v1.62.1/go.mod h1:iXhATfJQLjG3NWy56a6WVU73lWOcdYVxsvwCgoPljuo=
+modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
+modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
+modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ=
+modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8=
 modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
 modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
-modernc.org/memory v1.9.1 h1:V/Z1solwAVmMW1yttq3nDdZPJqV1rM05Ccq6KMSZ34g=
-modernc.org/memory v1.9.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
+modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
 modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
 modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
 modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
 modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
-modernc.org/sqlite v1.37.0 h1:s1TMe7T3Q3ovQiK2Ouz4Jwh7dw4ZDqbebSDTlSJdfjI=
-modernc.org/sqlite v1.37.0/go.mod h1:5YiWv+YviqGMuGw4V+PNplcyaJ5v+vQd7TQOgkACoJM=
+modernc.org/sqlite v1.38.2 h1:Aclu7+tgjgcQVShZqim41Bbw9Cho0y/7WzYptXqkEek=
+modernc.org/sqlite v1.38.2/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E=
 modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
 modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
 modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=

main.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/TwiN/gatus/v5/config"
 	"github.com/TwiN/gatus/v5/controller"
+	"github.com/TwiN/gatus/v5/metrics"
 	"github.com/TwiN/gatus/v5/storage/store"
 	"github.com/TwiN/gatus/v5/watchdog"
 	"github.com/TwiN/logr"
@@ -49,6 +50,7 @@ func main() {
 
 func start(cfg *config.Config) {
 	go controller.Handle(cfg)
+	metrics.InitializePrometheusMetrics(cfg, nil)
 	watchdog.Monitor(cfg)
 	go listenToConfigurationFileChanges(cfg)
 }
@@ -56,6 +58,7 @@ func start(cfg *config.Config) {
 func stop(cfg *config.Config) {
 	watchdog.Shutdown(cfg)
 	controller.Shutdown()
+	metrics.UnregisterPrometheusMetrics()
 }
 
 func save() {

metrics/metrics.go

@@ -3,82 +3,146 @@ package metrics
 import (
 	"strconv"
 
+	"github.com/TwiN/gatus/v5/config"
 	"github.com/TwiN/gatus/v5/config/endpoint"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 const namespace = "gatus" // The prefix of the metrics
 
 var (
-	initializedMetrics bool // Whether the metrics have been initialized
-
 	resultTotal                        *prometheus.CounterVec
 	resultDurationSeconds              *prometheus.GaugeVec
 	resultConnectedTotal               *prometheus.CounterVec
 	resultCodeTotal                    *prometheus.CounterVec
 	resultCertificateExpirationSeconds *prometheus.GaugeVec
 	resultEndpointSuccess              *prometheus.GaugeVec
+
+	// Track if metrics have been initialized to prevent duplicate registration
+	metricsInitialized bool
+	currentRegisterer  prometheus.Registerer
 )
 
-func initializePrometheusMetrics() {
-	resultTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+// UnregisterPrometheusMetrics unregisters all previously registered metrics
+func UnregisterPrometheusMetrics() {
+	if !metricsInitialized || currentRegisterer == nil {
+		return
+	}
+
+	// Unregister all metrics if they exist
+	if resultTotal != nil {
+		currentRegisterer.Unregister(resultTotal)
+	}
+	if resultDurationSeconds != nil {
+		currentRegisterer.Unregister(resultDurationSeconds)
+	}
+	if resultConnectedTotal != nil {
+		currentRegisterer.Unregister(resultConnectedTotal)
+	}
+	if resultCodeTotal != nil {
+		currentRegisterer.Unregister(resultCodeTotal)
+	}
+	if resultCertificateExpirationSeconds != nil {
+		currentRegisterer.Unregister(resultCertificateExpirationSeconds)
+	}
+	if resultEndpointSuccess != nil {
+		currentRegisterer.Unregister(resultEndpointSuccess)
+	}
+
+	metricsInitialized = false
+	currentRegisterer = nil
+}
+
+func InitializePrometheusMetrics(cfg *config.Config, reg prometheus.Registerer) {
+	// If metrics are already initialized, unregister them first
+	if metricsInitialized {
+		UnregisterPrometheusMetrics()
+	}
+
+	if reg == nil {
+		reg = prometheus.DefaultRegisterer
+	}
+
+	// Store the registerer for later unregistration
+	currentRegisterer = reg
+
+	extraLabels := cfg.GetUniqueExtraMetricLabels()
+	resultTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: namespace,
 		Name:      "results_total",
 		Help:      "Number of results per endpoint",
-	}, []string{"key", "group", "name", "type", "success"})
-	resultDurationSeconds = promauto.NewGaugeVec(prometheus.GaugeOpts{
+	}, append([]string{"key", "group", "name", "type", "success"}, extraLabels...))
+	reg.MustRegister(resultTotal)
+
+	resultDurationSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: namespace,
 		Name:      "results_duration_seconds",
 		Help:      "Duration of the request in seconds",
-	}, []string{"key", "group", "name", "type"})
-	resultConnectedTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+	}, append([]string{"key", "group", "name", "type"}, extraLabels...))
+	reg.MustRegister(resultDurationSeconds)
+
+	resultConnectedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: namespace,
 		Name:      "results_connected_total",
 		Help:      "Total number of results in which a connection was successfully established",
-	}, []string{"key", "group", "name", "type"})
-	resultCodeTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+	}, append([]string{"key", "group", "name", "type"}, extraLabels...))
+	reg.MustRegister(resultConnectedTotal)
+
+	resultCodeTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: namespace,
 		Name:      "results_code_total",
 		Help:      "Total number of results by code",
-	}, []string{"key", "group", "name", "type", "code"})
-	resultCertificateExpirationSeconds = promauto.NewGaugeVec(prometheus.GaugeOpts{
+	}, append([]string{"key", "group", "name", "type", "code"}, extraLabels...))
+	reg.MustRegister(resultCodeTotal)
+
+	resultCertificateExpirationSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: namespace,
 		Name:      "results_certificate_expiration_seconds",
 		Help:      "Number of seconds until the certificate expires",
-	}, []string{"key", "group", "name", "type"})
-	resultEndpointSuccess = promauto.NewGaugeVec(prometheus.GaugeOpts{
+	}, append([]string{"key", "group", "name", "type"}, extraLabels...))
+	reg.MustRegister(resultCertificateExpirationSeconds)
+
+	resultEndpointSuccess = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: namespace,
 		Name:      "results_endpoint_success",
 		Help:      "Displays whether or not the endpoint was a success",
-	}, []string{"key", "group", "name", "type"})
+	}, append([]string{"key", "group", "name", "type"}, extraLabels...))
+	reg.MustRegister(resultEndpointSuccess)
+
+	// Mark as initialized
+	metricsInitialized = true
 }
 
 // PublishMetricsForEndpoint publishes metrics for the given endpoint and its result.
 // These metrics will be exposed at /metrics if the metrics are enabled
-func PublishMetricsForEndpoint(ep *endpoint.Endpoint, result *endpoint.Result) {
-	if !initializedMetrics {
-		initializePrometheusMetrics()
-		initializedMetrics = true
+func PublishMetricsForEndpoint(ep *endpoint.Endpoint, result *endpoint.Result, extraLabels []string) {
+	labelValues := []string{}
+	for _, label := range extraLabels {
+		if value, ok := ep.ExtraLabels[label]; ok {
+			labelValues = append(labelValues, value)
+		} else {
+			labelValues = append(labelValues, "")
+		}
 	}
+
 	endpointType := ep.Type()
-	resultTotal.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType), strconv.FormatBool(result.Success)).Inc()
-	resultDurationSeconds.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType)).Set(result.Duration.Seconds())
+	resultTotal.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType), strconv.FormatBool(result.Success)}, labelValues...)...).Inc()
+	resultDurationSeconds.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType)}, labelValues...)...).Set(result.Duration.Seconds())
 	if result.Connected {
-		resultConnectedTotal.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType)).Inc()
+		resultConnectedTotal.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType)}, labelValues...)...).Inc()
 	}
 	if result.DNSRCode != "" {
-		resultCodeTotal.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType), result.DNSRCode).Inc()
+		resultCodeTotal.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType), result.DNSRCode}, labelValues...)...).Inc()
 	}
 	if result.HTTPStatus != 0 {
-		resultCodeTotal.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType), strconv.Itoa(result.HTTPStatus)).Inc()
+		resultCodeTotal.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType), strconv.Itoa(result.HTTPStatus)}, labelValues...)...).Inc()
 	}
 	if result.CertificateExpiration != 0 {
-		resultCertificateExpirationSeconds.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType)).Set(result.CertificateExpiration.Seconds())
+		resultCertificateExpirationSeconds.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType)}, labelValues...)...).Set(result.CertificateExpiration.Seconds())
 	}
 	if result.Success {
-		resultEndpointSuccess.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType)).Set(1)
+		resultEndpointSuccess.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType)}, labelValues...)...).Set(1)
 	} else {
-		resultEndpointSuccess.WithLabelValues(ep.Key(), ep.Group, ep.Name, string(endpointType)).Set(0)
+		resultEndpointSuccess.WithLabelValues(append([]string{ep.Key(), ep.Group, ep.Name, string(endpointType)}, labelValues...)...).Set(0)
 	}
 }

metrics/metrics_test.go

@@ -5,13 +5,110 @@ import (
 	"testing"
 	"time"
 
+	"github.com/TwiN/gatus/v5/config"
 	"github.com/TwiN/gatus/v5/config/endpoint"
 	"github.com/TwiN/gatus/v5/config/endpoint/dns"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/testutil"
 )
 
+// TestInitializePrometheusMetrics tests metrics initialization with extraLabels.
+// Note: Because of the global Prometheus registry, this test can only safely verify one label set per process.
+// If the function is called with a different set of labels for the same metric, a panic will occur.
+func TestInitializePrometheusMetrics(t *testing.T) {
+	cfgWithExtras := &config.Config{
+		Endpoints: []*endpoint.Endpoint{
+			{
+				Name:  "TestEP",
+				Group: "G",
+				URL:   "http://x/",
+				ExtraLabels: map[string]string{
+					"foo":   "foo-val",
+					"hello": "world-val",
+				},
+			},
+		},
+	}
+	reg := prometheus.NewRegistry()
+	InitializePrometheusMetrics(cfgWithExtras, reg)
+	// Metrics variables should be non-nil
+	if resultTotal == nil {
+		t.Error("resultTotal metric not initialized")
+	}
+	if resultDurationSeconds == nil {
+		t.Error("resultDurationSeconds metric not initialized")
+	}
+	if resultConnectedTotal == nil {
+		t.Error("resultConnectedTotal metric not initialized")
+	}
+	if resultCodeTotal == nil {
+		t.Error("resultCodeTotal metric not initialized")
+	}
+	if resultCertificateExpirationSeconds == nil {
+		t.Error("resultCertificateExpirationSeconds metric not initialized")
+	}
+	if resultEndpointSuccess == nil {
+		t.Error("resultEndpointSuccess metric not initialized")
+	}
+
+	defer func() {
+		if r := recover(); r != nil {
+			t.Errorf("resultTotal.WithLabelValues panicked: %v", r)
+		}
+	}()
+	_ = resultTotal.WithLabelValues("k", "g", "n", "ty", "true", "fval", "hval")
+}
+
+// TestPublishMetricsForEndpoint_withExtraLabels ensures extraLabels are included in the exported metrics.
+func TestPublishMetricsForEndpoint_withExtraLabels(t *testing.T) {
+	// Only test one label set per process due to Prometheus registry limits.
+	reg := prometheus.NewRegistry()
+	InitializePrometheusMetrics(&config.Config{
+		Endpoints: []*endpoint.Endpoint{
+			{
+				Name: "ep-extra",
+				URL:  "https://sample.com",
+				ExtraLabels: map[string]string{
+					"foo": "my-foo",
+					"bar": "my-bar",
+				},
+			},
+		},
+	}, reg)
+
+	ep := &endpoint.Endpoint{
+		Name:  "ep-extra",
+		Group: "g1",
+		URL:   "https://sample.com",
+		ExtraLabels: map[string]string{
+			"foo": "my-foo",
+			"bar": "my-bar",
+		},
+	}
+	result := &endpoint.Result{
+		HTTPStatus: 200,
+		Connected:  true,
+		Duration:   2340 * time.Millisecond,
+		Success:    true,
+	}
+	// Order of extraLabels as per GetUniqueExtraMetricLabels is ["foo", "bar"]
+	PublishMetricsForEndpoint(ep, result, []string{"foo", "bar"})
+
+	expected := `
+# HELP gatus_results_total Number of results per endpoint
+# TYPE gatus_results_total counter
+gatus_results_total{bar="my-bar",foo="my-foo",group="g1",key="g1_ep-extra",name="ep-extra",success="true",type="HTTP"} 1
+`
+	err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expected), "gatus_results_total")
+	if err != nil {
+		t.Error("metrics export does not include extraLabels as expected:", err)
+	}
+}
+
 func TestPublishMetricsForEndpoint(t *testing.T) {
+	reg := prometheus.NewRegistry()
+	InitializePrometheusMetrics(&config.Config{}, reg)
+
 	httpEndpoint := &endpoint.Endpoint{Name: "http-ep-name", Group: "http-ep-group", URL: "https://example.org"}
 	PublishMetricsForEndpoint(httpEndpoint, &endpoint.Result{
 		HTTPStatus: 200,
@@ -23,8 +120,8 @@ func TestPublishMetricsForEndpoint(t *testing.T) {
 		},
 		Success:               true,
 		CertificateExpiration: 49 * time.Hour,
-	})
-	err := testutil.GatherAndCompare(prometheus.Gatherers{prometheus.DefaultGatherer}, bytes.NewBufferString(`
+	}, []string{})
+	err := testutil.GatherAndCompare(reg, bytes.NewBufferString(`
 # HELP gatus_results_code_total Total number of results by code
 # TYPE gatus_results_code_total counter
 gatus_results_code_total{code="200",group="http-ep-group",key="http-ep-group_http-ep-name",name="http-ep-name",type="HTTP"} 1
@@ -57,8 +154,8 @@ gatus_results_endpoint_success{group="http-ep-group",key="http-ep-group_http-ep-
 		},
 		Success:               false,
 		CertificateExpiration: 47 * time.Hour,
-	})
-	err = testutil.GatherAndCompare(prometheus.Gatherers{prometheus.DefaultGatherer}, bytes.NewBufferString(`
+	}, []string{})
+	err = testutil.GatherAndCompare(reg, bytes.NewBufferString(`
 # HELP gatus_results_code_total Total number of results by code
 # TYPE gatus_results_code_total counter
 gatus_results_code_total{code="200",group="http-ep-group",key="http-ep-group_http-ep-name",name="http-ep-name",type="HTTP"} 2
@@ -82,10 +179,12 @@ gatus_results_endpoint_success{group="http-ep-group",key="http-ep-group_http-ep-
 	if err != nil {
 		t.Errorf("Expected no errors but got: %v", err)
 	}
-	dnsEndpoint := &endpoint.Endpoint{Name: "dns-ep-name", Group: "dns-ep-group", URL: "8.8.8.8", DNSConfig: &dns.Config{
-		QueryType: "A",
-		QueryName: "example.com.",
-	}}
+	dnsEndpoint := &endpoint.Endpoint{
+		Name: "dns-ep-name", Group: "dns-ep-group", URL: "8.8.8.8", DNSConfig: &dns.Config{
+			QueryType: "A",
+			QueryName: "example.com.",
+		},
+	}
 	PublishMetricsForEndpoint(dnsEndpoint, &endpoint.Result{
 		DNSRCode:  "NOERROR",
 		Connected: true,
@@ -94,8 +193,8 @@ gatus_results_endpoint_success{group="http-ep-group",key="http-ep-group_http-ep-
 			{Condition: "[DNS_RCODE] == NOERROR", Success: true},
 		},
 		Success: true,
-	})
-	err = testutil.GatherAndCompare(prometheus.Gatherers{prometheus.DefaultGatherer}, bytes.NewBufferString(`
+	}, []string{})
+	err = testutil.GatherAndCompare(reg, bytes.NewBufferString(`
 # HELP gatus_results_code_total Total number of results by code
 # TYPE gatus_results_code_total counter
 gatus_results_code_total{code="200",group="http-ep-group",key="http-ep-group_http-ep-name",name="http-ep-name",type="HTTP"} 2

storage/store/memory/memory.go

@@ -211,6 +211,23 @@ func (s *Store) DeleteAllTriggeredAlertsNotInChecksumsByEndpoint(ep *endpoint.En
 	return 0
 }
 
+// HasEndpointStatusNewerThan checks whether an endpoint has a status newer than the provided timestamp
+func (s *Store) HasEndpointStatusNewerThan(key string, timestamp time.Time) (bool, error) {
+	s.RLock()
+	defer s.RUnlock()
+	endpointStatus := s.cache.GetValue(key)
+	if endpointStatus == nil {
+		// If no endpoint exists, there's no newer status, so return false instead of an error
+		return false, nil
+	}
+	for _, result := range endpointStatus.(*endpoint.Status).Results {
+		if result.Timestamp.After(timestamp) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
 // Clear deletes everything from the store
 func (s *Store) Clear() {
 	s.cache.Clear()

storage/store/memory/memory_test.go

@@ -84,6 +84,7 @@ var (
 // This test is simply an extra sanity check
 func TestStore_SanityCheck(t *testing.T) {
 	store, _ := NewStore(storage.DefaultMaximumNumberOfResults, storage.DefaultMaximumNumberOfEvents)
+	defer store.Clear()
 	defer store.Close()
 	store.Insert(&testEndpoint, &testSuccessfulResult)
 	endpointStatuses, _ := store.GetAllEndpointStatuses(paging.NewEndpointStatusParams())
@@ -134,3 +135,30 @@ func TestStore_Save(t *testing.T) {
 	store.Clear()
 	store.Close()
 }
+
+func TestStore_HasEndpointStatusNewerThan(t *testing.T) {
+	store, _ := NewStore(storage.DefaultMaximumNumberOfResults, storage.DefaultMaximumNumberOfEvents)
+	defer store.Clear()
+	defer store.Close()
+	// Insert a result
+	err := store.Insert(&testEndpoint, &testSuccessfulResult)
+	if err != nil {
+		t.Fatalf("expected no error while inserting result, got %v", err)
+	}
+	// Check with a timestamp in the past
+	hasNewerStatus, err := store.HasEndpointStatusNewerThan(testEndpoint.Key(), time.Now().Add(-time.Hour))
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+	if !hasNewerStatus {
+		t.Fatal("expected to have a newer status, but didn't")
+	}
+	// Check with a timestamp in the future
+	hasNewerStatus, err = store.HasEndpointStatusNewerThan(testEndpoint.Key(), time.Now().Add(time.Hour))
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+	if hasNewerStatus {
+		t.Fatal("expected not to have a newer status, but did")
+	}
+}

storage/store/sql/sql.go

@@ -514,6 +514,24 @@ func (s *Store) DeleteAllTriggeredAlertsNotInChecksumsByEndpoint(ep *endpoint.En
 	return int(rowsAffects)
 }
 
+// HasEndpointStatusNewerThan checks whether an endpoint has a status newer than the provided timestamp
+func (s *Store) HasEndpointStatusNewerThan(key string, timestamp time.Time) (bool, error) {
+	if timestamp.IsZero() {
+		return false, errors.New("timestamp is zero")
+	}
+	var count int
+	err := s.db.QueryRow(
+		"SELECT COUNT(*) FROM endpoint_results WHERE endpoint_id = (SELECT endpoint_id FROM endpoints WHERE endpoint_key = $1 LIMIT 1) AND timestamp > $2",
+		key,
+		timestamp.UTC(),
+	).Scan(&count)
+	if err != nil {
+		// If the endpoint doesn't exist, we return false instead of an error
+		return false, nil
+	}
+	return count > 0, nil
+}
+
 // Clear deletes everything from the store
 func (s *Store) Clear() {
 	_, _ = s.db.Exec("DELETE FROM endpoints")

storage/store/sql/sql_test.go

@@ -853,3 +853,36 @@ func TestStore_DeleteAllTriggeredAlertsNotInChecksumsByEndpoint(t *testing.T) {
 		t.Error("expected alert3 to exist for ep2")
 	}
 }
+
+func TestStore_HasEndpointStatusNewerThan(t *testing.T) {
+	store, _ := NewStore("sqlite", t.TempDir()+"/TestStore_HasEndpointStatusNewerThan.db", false, storage.DefaultMaximumNumberOfResults, storage.DefaultMaximumNumberOfEvents)
+	defer store.Close()
+	// Insert an endpoint status
+	if err := store.Insert(&testEndpoint, &testSuccessfulResult); err != nil {
+		t.Fatal("expected no error, got", err.Error())
+	}
+	// Check if it has a status newer than 1 hour ago
+	hasNewerStatus, err := store.HasEndpointStatusNewerThan(testEndpoint.Key(), time.Now().Add(-time.Hour))
+	if err != nil {
+		t.Fatal("expected no error, got", err.Error())
+	}
+	if !hasNewerStatus {
+		t.Error("expected to have a newer status")
+	}
+	// Check if it has a status newer than 2 days ago
+	hasNewerStatus, err = store.HasEndpointStatusNewerThan(testEndpoint.Key(), time.Now().Add(-48*time.Hour))
+	if err != nil {
+		t.Fatal("expected no error, got", err.Error())
+	}
+	if !hasNewerStatus {
+		t.Error("expected to have a newer status")
+	}
+	// Check if there's a status newer than 1 hour in the future (silly test, but it should work)
+	hasNewerStatus, err = store.HasEndpointStatusNewerThan(testEndpoint.Key(), time.Now().Add(time.Hour))
+	if err != nil {
+		t.Fatal("expected no error, got", err.Error())
+	}
+	if hasNewerStatus {
+		t.Error("expected not to have a newer status in the future")
+	}
+}

storage/store/store.go

@@ -57,6 +57,9 @@ type Store interface {
 	// This prevents triggered alerts that have been removed or modified from lingering in the database.
 	DeleteAllTriggeredAlertsNotInChecksumsByEndpoint(ep *endpoint.Endpoint, checksums []string) int
 
+	// HasEndpointStatusNewerThan checks whether an endpoint has a status newer than the provided timestamp
+	HasEndpointStatusNewerThan(key string, timestamp time.Time) (bool, error)
+
 	// Clear deletes everything from the store
 	Clear()
 

watchdog/alerting.go

@@ -2,7 +2,9 @@ package watchdog
 
 import (
 	"errors"
+	"log"
 	"os"
+	"time"
 
 	"github.com/TwiN/gatus/v5/alerting"
 	"github.com/TwiN/gatus/v5/config/endpoint"
@@ -30,14 +32,24 @@ func handleAlertsToTrigger(ep *endpoint.Endpoint, result *endpoint.Result, alert
 		if !endpointAlert.IsEnabled() || endpointAlert.FailureThreshold > ep.NumberOfFailuresInARow {
 			continue
 		}
-		if endpointAlert.Triggered {
-			logr.Debugf("[watchdog.handleAlertsToTrigger] Alert for endpoint with key=%s with description='%s' has already been TRIGGERED, skipping", ep.Key(), endpointAlert.GetDescription())
+		// Determine if an initial alert should be sent
+		sendInitialAlert := !endpointAlert.Triggered
+		// Determine if a reminder should be sent
+		sendReminder := endpointAlert.Triggered && endpointAlert.MinimumReminderInterval > 0 && time.Since(ep.LastReminderSent) >= endpointAlert.MinimumReminderInterval
+		// If neither initial alert nor reminder needs to be sent, skip to the next alert
+		if !sendInitialAlert && !sendReminder {
+			logr.Debugf("[watchdog.handleAlertsToTrigger] Alert for endpoint=%s with description='%s' is not due for triggering or reminding, skipping", ep.Name, endpointAlert.GetDescription())
 			continue
 		}
 		alertProvider := alertingConfig.GetAlertingProviderByAlertType(endpointAlert.Type)
 		if alertProvider != nil {
 			logr.Infof("[watchdog.handleAlertsToTrigger] Sending %s alert because alert for endpoint with key=%s with description='%s' has been TRIGGERED", endpointAlert.Type, ep.Key(), endpointAlert.GetDescription())
 			var err error
+			alertType := "reminder"
+			if sendInitialAlert {
+				alertType = "initial"
+			}
+			log.Printf("[watchdog.handleAlertsToTrigger] Sending %s %s alert because alert for endpoint=%s with description='%s' has been TRIGGERED", alertType, endpointAlert.Type, ep.Name, endpointAlert.GetDescription())
 			if os.Getenv("MOCK_ALERT_PROVIDER") == "true" {
 				if os.Getenv("MOCK_ALERT_PROVIDER_ERROR") == "true" {
 					err = errors.New("error")
@@ -48,7 +60,11 @@ func handleAlertsToTrigger(ep *endpoint.Endpoint, result *endpoint.Result, alert
 			if err != nil {
 				logr.Errorf("[watchdog.handleAlertsToTrigger] Failed to send an alert for endpoint with key=%s: %s", ep.Key(), err.Error())
 			} else {
-				endpointAlert.Triggered = true
+				// Mark initial alert as triggered and update last reminder time
+				if sendInitialAlert {
+					endpointAlert.Triggered = true
+				}
+				ep.LastReminderSent = time.Now()
 				if err := store.Get().UpsertTriggeredEndpointAlert(ep, endpointAlert); err != nil {
 					logr.Errorf("[watchdog.handleAlertsToTrigger] Failed to persist triggered endpoint alert for endpoint with key=%s: %s", ep.Key(), err.Error())
 				}

watchdog/alerting_test.go

@@ -3,6 +3,7 @@ package watchdog
 import (
 	"os"
 	"testing"
+	"time"
 
 	"github.com/TwiN/gatus/v5/alerting"
 	"github.com/TwiN/gatus/v5/alerting/alert"
@@ -517,6 +518,48 @@ func TestHandleAlertingWithProviderThatOnlyReturnsErrorOnResolve(t *testing.T) {
 	verify(t, ep, 0, 2, false, "")
 }
 
+func TestHandleAlertingWithMinimumReminderInterval(t *testing.T) {
+	_ = os.Setenv("MOCK_ALERT_PROVIDER", "true")
+	defer os.Clearenv()
+
+	cfg := &config.Config{
+		Alerting: &alerting.Config{
+			Custom: &custom.AlertProvider{
+				DefaultConfig: custom.Config{
+					URL:    "https://twin.sh/health",
+					Method: "GET",
+				},
+			},
+		},
+	}
+	enabled := true
+	ep := &endpoint.Endpoint{
+		URL: "https://example.com",
+		Alerts: []*alert.Alert{
+			{
+				Type:                    alert.TypeCustom,
+				Enabled:                 &enabled,
+				FailureThreshold:        2,
+				SuccessThreshold:        3,
+				SendOnResolved:          &enabled,
+				Triggered:               false,
+				MinimumReminderInterval: 1 * time.Second,
+			},
+		},
+	}
+
+	verify(t, ep, 0, 0, false, "The alert shouldn't start triggered")
+	HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting)
+	verify(t, ep, 1, 0, false, "The alert shouldn't have triggered")
+	HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting)
+	verify(t, ep, 2, 0, true, "The alert should've triggered")
+	HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting)
+	verify(t, ep, 3, 0, true, "The alert should still be triggered")
+	HandleAlerting(ep, &endpoint.Result{Success: false}, cfg.Alerting)
+	verify(t, ep, 4, 0, true, "The alert should still be triggered")
+	HandleAlerting(ep, &endpoint.Result{Success: true}, cfg.Alerting)
+}
+
 func verify(t *testing.T, ep *endpoint.Endpoint, expectedNumberOfFailuresInARow, expectedNumberOfSuccessInARow int, expectedTriggered bool, expectedTriggeredReason string) {
 	if ep.NumberOfFailuresInARow != expectedNumberOfFailuresInARow {
 		t.Errorf("endpoint.NumberOfFailuresInARow should've been %d, got %d", expectedNumberOfFailuresInARow, ep.NumberOfFailuresInARow)

watchdog/watchdog.go

@@ -27,19 +27,28 @@ var (
 // Monitor loops over each endpoint and starts a goroutine to monitor each endpoint separately
 func Monitor(cfg *config.Config) {
 	ctx, cancelFunc = context.WithCancel(context.Background())
+	extraLabels := cfg.GetUniqueExtraMetricLabels()
 	for _, endpoint := range cfg.Endpoints {
 		if endpoint.IsEnabled() {
 			// To prevent multiple requests from running at the same time, we'll wait for a little before each iteration
 			time.Sleep(777 * time.Millisecond)
-			go monitor(endpoint, cfg.Alerting, cfg.Maintenance, cfg.Connectivity, cfg.DisableMonitoringLock, cfg.Metrics, ctx)
+			go monitor(endpoint, cfg.Alerting, cfg.Maintenance, cfg.Connectivity, cfg.DisableMonitoringLock, cfg.Metrics, extraLabels, ctx)
+		}
+	}
+	for _, externalEndpoint := range cfg.ExternalEndpoints {
+		// Check if the external endpoint is enabled and is using heartbeat
+		// If the external endpoint does not use heartbeat, then it does not need to be monitored periodically, because
+		// alerting is checked every time an external endpoint is pushed to Gatus, unlike normal endpoints.
+		if externalEndpoint.IsEnabled() && externalEndpoint.Heartbeat.Interval > 0 {
+			go monitorExternalEndpointHeartbeat(externalEndpoint, cfg.Alerting, cfg.Maintenance, cfg.Connectivity, cfg.DisableMonitoringLock, cfg.Metrics, ctx, extraLabels)
 		}
 	}
 }
 
 // monitor a single endpoint in a loop
-func monitor(ep *endpoint.Endpoint, alertingConfig *alerting.Config, maintenanceConfig *maintenance.Config, connectivityConfig *connectivity.Config, disableMonitoringLock bool, enabledMetrics bool, ctx context.Context) {
+func monitor(ep *endpoint.Endpoint, alertingConfig *alerting.Config, maintenanceConfig *maintenance.Config, connectivityConfig *connectivity.Config, disableMonitoringLock bool, enabledMetrics bool, extraLabels []string, ctx context.Context) {
 	// Run it immediately on start
-	execute(ep, alertingConfig, maintenanceConfig, connectivityConfig, disableMonitoringLock, enabledMetrics)
+	execute(ep, alertingConfig, maintenanceConfig, connectivityConfig, disableMonitoringLock, enabledMetrics, extraLabels)
 	// Loop for the next executions
 	ticker := time.NewTicker(ep.Interval)
 	defer ticker.Stop()
@@ -49,7 +58,7 @@ func monitor(ep *endpoint.Endpoint, alertingConfig *alerting.Config, maintenance
 			logr.Warnf("[watchdog.monitor] Canceling current execution of group=%s; endpoint=%s; key=%s", ep.Group, ep.Name, ep.Key())
 			return
 		case <-ticker.C:
-			execute(ep, alertingConfig, maintenanceConfig, connectivityConfig, disableMonitoringLock, enabledMetrics)
+			execute(ep, alertingConfig, maintenanceConfig, connectivityConfig, disableMonitoringLock, enabledMetrics, extraLabels)
 		}
 	}
 	// Just in case somebody wandered all the way to here and wonders, "what about ExternalEndpoints?"
@@ -57,7 +66,7 @@ func monitor(ep *endpoint.Endpoint, alertingConfig *alerting.Config, maintenance
 	// periodically like they are for normal endpoints.
 }
 
-func execute(ep *endpoint.Endpoint, alertingConfig *alerting.Config, maintenanceConfig *maintenance.Config, connectivityConfig *connectivity.Config, disableMonitoringLock bool, enabledMetrics bool) {
+func execute(ep *endpoint.Endpoint, alertingConfig *alerting.Config, maintenanceConfig *maintenance.Config, connectivityConfig *connectivity.Config, disableMonitoringLock bool, enabledMetrics bool, extraLabels []string) {
 	if !disableMonitoringLock {
 		// By placing the lock here, we prevent multiple endpoints from being monitored at the exact same time, which
 		// could cause performance issues and return inaccurate results
@@ -72,7 +81,7 @@ func execute(ep *endpoint.Endpoint, alertingConfig *alerting.Config, maintenance
 	logr.Debugf("[watchdog.execute] Monitoring group=%s; endpoint=%s; key=%s", ep.Group, ep.Name, ep.Key())
 	result := ep.EvaluateHealth()
 	if enabledMetrics {
-		metrics.PublishMetricsForEndpoint(ep, result)
+		metrics.PublishMetricsForEndpoint(ep, result, extraLabels)
 	}
 	UpdateEndpointStatuses(ep, result)
 	if logr.GetThreshold() == logr.LevelDebug && !result.Success {
@@ -96,6 +105,76 @@ func execute(ep *endpoint.Endpoint, alertingConfig *alerting.Config, maintenance
 	logr.Debugf("[watchdog.execute] Waiting for interval=%s before monitoring group=%s endpoint=%s (key=%s) again", ep.Interval, ep.Group, ep.Name, ep.Key())
 }
 
+func monitorExternalEndpointHeartbeat(ee *endpoint.ExternalEndpoint, alertingConfig *alerting.Config, maintenanceConfig *maintenance.Config, connectivityConfig *connectivity.Config, disableMonitoringLock bool, enabledMetrics bool, ctx context.Context, extraLabels []string) {
+	ticker := time.NewTicker(ee.Heartbeat.Interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			logr.Warnf("[watchdog.monitorExternalEndpointHeartbeat] Canceling current execution of group=%s; endpoint=%s; key=%s", ee.Group, ee.Name, ee.Key())
+			return
+		case <-ticker.C:
+			executeExternalEndpointHeartbeat(ee, alertingConfig, maintenanceConfig, connectivityConfig, disableMonitoringLock, enabledMetrics, extraLabels)
+		}
+	}
+}
+
+func executeExternalEndpointHeartbeat(ee *endpoint.ExternalEndpoint, alertingConfig *alerting.Config, maintenanceConfig *maintenance.Config, connectivityConfig *connectivity.Config, disableMonitoringLock bool, enabledMetrics bool, extraLabels []string) {
+	if !disableMonitoringLock {
+		// By placing the lock here, we prevent multiple endpoints from being monitored at the exact same time, which
+		// could cause performance issues and return inaccurate results
+		monitoringMutex.Lock()
+		defer monitoringMutex.Unlock()
+	}
+	// If there's a connectivity checker configured, check if Gatus has internet connectivity
+	if connectivityConfig != nil && connectivityConfig.Checker != nil && !connectivityConfig.Checker.IsConnected() {
+		logr.Infof("[watchdog.monitorExternalEndpointHeartbeat] No connectivity; skipping execution")
+		return
+	}
+	logr.Debugf("[watchdog.monitorExternalEndpointHeartbeat] Checking heartbeat for group=%s; endpoint=%s; key=%s", ee.Group, ee.Name, ee.Key())
+	convertedEndpoint := ee.ToEndpoint()
+	hasReceivedResultWithinHeartbeatInterval, err := store.Get().HasEndpointStatusNewerThan(ee.Key(), time.Now().Add(-ee.Heartbeat.Interval))
+	if err != nil {
+		logr.Errorf("[watchdog.monitorExternalEndpointHeartbeat] Failed to check if endpoint has received a result within the heartbeat interval: %s", err.Error())
+		return
+	}
+	if hasReceivedResultWithinHeartbeatInterval {
+		// If we received a result within the heartbeat interval, we don't want to create a successful result, so we
+		// skip the rest. We don't have to worry about alerting or metrics, because if the previous heartbeat failed
+		// while this one succeeds, it implies that there was a new result pushed, and that result being pushed
+		// should've resolved the alert.
+		logr.Infof("[watchdog.monitorExternalEndpointHeartbeat] Checked heartbeat for group=%s; endpoint=%s; key=%s; success=%v; errors=%d", ee.Group, ee.Name, ee.Key(), hasReceivedResultWithinHeartbeatInterval, 0)
+		return
+	}
+	// All code after this point assumes the heartbeat failed
+	result := &endpoint.Result{
+		Timestamp: time.Now(),
+		Success:   false,
+		Errors:    []string{"heartbeat: no update received within " + ee.Heartbeat.Interval.String()},
+	}
+	if enabledMetrics {
+		metrics.PublishMetricsForEndpoint(convertedEndpoint, result, extraLabels)
+	}
+	UpdateEndpointStatuses(convertedEndpoint, result)
+	logr.Infof("[watchdog.monitorExternalEndpointHeartbeat] Checked heartbeat for group=%s; endpoint=%s; key=%s; success=%v; errors=%d; duration=%s", ee.Group, ee.Name, ee.Key(), result.Success, len(result.Errors), result.Duration.Round(time.Millisecond))
+	inEndpointMaintenanceWindow := false
+	for _, maintenanceWindow := range ee.MaintenanceWindows {
+		if maintenanceWindow.IsUnderMaintenance() {
+			logr.Debug("[watchdog.monitorExternalEndpointHeartbeat] Under endpoint maintenance window")
+			inEndpointMaintenanceWindow = true
+		}
+	}
+	if !maintenanceConfig.IsUnderMaintenance() && !inEndpointMaintenanceWindow {
+		HandleAlerting(convertedEndpoint, result, alertingConfig)
+		// Sync the failure/success counters back to the external endpoint
+		ee.NumberOfSuccessesInARow = convertedEndpoint.NumberOfSuccessesInARow
+		ee.NumberOfFailuresInARow = convertedEndpoint.NumberOfFailuresInARow
+	} else {
+		logr.Debug("[watchdog.monitorExternalEndpointHeartbeat] Not handling alerting because currently in the maintenance window")
+	}
+	logr.Debugf("[watchdog.monitorExternalEndpointHeartbeat] Waiting for interval=%s before checking heartbeat for group=%s endpoint=%s (key=%s) again", ee.Heartbeat.Interval, ee.Group, ee.Name, ee.Key())
+}
+
 // UpdateEndpointStatuses updates the slice of endpoint statuses
 func UpdateEndpointStatuses(ep *endpoint.Endpoint, result *endpoint.Result) {
 	if err := store.Get().Insert(ep, result); err != nil {