Merge pull request #10 from TwinProduction/notify-on-resolved

Support sending an alert when an unhealthy service becomes healthy again

.github/workflows/build.yml

@@ -0,0 +1,27 @@
+name: build
+on:
+  pull_request:
+    paths-ignore:
+      - '*.md'
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - '*.md'
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Set up Go 1.14
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.14
+        id: go
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v1
+      - name: Build binary to make sure it works
+        run: go build -mod vendor
+      - name: Test
+        run: go test -mod vendor -cover ./...

.github/workflows/go.yml

@@ -1,34 +0,0 @@
-name: Go
-on: [push]
-jobs:
-  build-112:
-    name: Build with Go 1.12
-    runs-on: ubuntu-latest
-    steps:
-    - name: Set up Go 1.12
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.12
-      id: go
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v1
-    - name: Test
-      run: go test -mod vendor ./...
-    - name: Build
-      run: go build -mod vendor
-
-  build-113:
-    name: Build with Go 1.13
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Go 1.13
-        uses: actions/setup-go@v1
-        with:
-          go-version: 1.13
-        id: go
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v1
-      - name: Test
-        run: go test -mod vendor ./...
-      - name: Build
-        run: go build -mod vendor

.gitignore

@@ -1,4 +1,2 @@
 bin
-.idea
-test.db
-daily.db
+.idea

LICENSE.md

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2020 TwinProduction
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -1,11 +1,41 @@
-# gatus
+![Gatus](static/logo-with-name.png)
 
+![build](https://github.com/TwinProduction/gatus/workflows/build/badge.svg?branch=master)
+[![Go Report Card](https://goreportcard.com/badge/github.com/TwinProduction/gatus)](https://goreportcard.com/report/github.com/TwinProduction/gatus)
 [![Docker pulls](https://img.shields.io/docker/pulls/twinproduction/gatus.svg)](https://cloud.docker.com/repository/docker/twinproduction/gatus)
 
 A service health dashboard in Go that is meant to be used as a docker 
 image with a custom configuration file.
 
-Live example: https://status.twinnation.org/
+I personally deploy it in my Kubernetes cluster and have it monitor the status of my
+core applications: https://status.twinnation.org/
+
+
+## Table of Contents
+
+- [Features](#features)
+- [Usage](#usage)
+  - [Configuration](#configuration)
+  - [Conditions](#conditions)
+- [Docker](#docker)
+- [Running the tests](#running-the-tests)
+- [Using in Production](#using-in-production)
+- [FAQ](#faq)
+  - [Sending a GraphQL request](#sending-a-graphql-request)
+  - [Configuring Slack alerts](#configuring-slack-alerts)
+  - [Configuring Twilio alerts](#configuring-twilio-alerts)
+  - [Configuring custom alerts](#configuring-custom-alerts)
+
+
+## Features
+
+The main features of Gatus are:
+- **Highly flexible health check conditions**: While checking the response status may be enough for some use cases, Gatus goes much further and allows you to add conditions on the response time, the response body and even the IP address.
+- **Ability to use Gatus for user acceptance tests**: Thanks to the point above, you can leverage this application to create automated user acceptance tests.
+- **Very easy to configure**: Not only is the configuration designed to be as readable as possible, it's also extremely easy to add a new service or a new endpoint to monitor.
+- **Alerting**: While having a pretty visual dashboard is useful to keep track of the state of your application(s), you probably don't want to stare at it all day. Thus, notifications via Slack are supported out of the box with the ability to configure a custom alerting provider for any needs you might have, whether it be a different provider like PagerDuty or a custom application that manages automated rollbacks. 
+- **Metrics**
+- **Low resource consumption**: As with most Go applications, the resource footprint that this application requires is negligibly small.
 
 
 ## Usage
@@ -14,26 +44,86 @@ By default, the configuration file is expected to be at `config/config.yaml`.
 
 You can specify a custom path by setting the `GATUS_CONFIG_FILE` environment variable.
 
+Here's a simple example:
+
 ```yaml
 metrics: true         # Whether to expose metrics at /metrics
 services:
   - name: twinnation  # Name of your service, can be anything
-    url: https://twinnation.org/health
-    interval: 15s     # Duration to wait between every status check (opt. default: 10s)
+    url: "https://twinnation.org/health"
+    interval: 30s     # Duration to wait between every status check (default: 60s)
     conditions:
-      - "[STATUS] == 200"
-  - name: github
-    url: https://api.github.com/healthz
+      - "[STATUS] == 200"         # Status must be 200
+      - "[BODY].status == UP"     # The json path "$.status" must be equal to UP
+      - "[RESPONSE_TIME] < 300"   # Response time must be under 300ms
+  - name: example
+    url: "https://example.org/"
+    interval: 30s
     conditions:
       - "[STATUS] == 200"
 ```
 
-Note that you can also add environment variables in the your configuration file (i.e. `$DOMAIN`, `${DOMAIN}`)
+This example would look like this:
+
+![Simple example](.github/assets/example.png)
+
+Note that you can also add environment variables in the configuration file (i.e. `$DOMAIN`, `${DOMAIN}`)
+
+
+### Configuration
+
+| Parameter                              | Description                                                     | Default        |
+| -------------------------------------- | --------------------------------------------------------------- | -------------- |
+| `debug`                                | Whether to enable debug logs                                    | `false`        |
+| `metrics`                              | Whether to expose metrics at /metrics                           | `false`        |
+| `services`                             | List of services to monitor                                     | Required `[]`  |
+| `services[].name`                      | Name of the service. Can be anything.                           | Required `""`  |
+| `services[].url`                       | URL to send the request to                                      | Required `""`  |
+| `services[].conditions`                | Conditions used to determine the health of the service          | `[]`           |
+| `services[].interval`                  | Duration to wait between every status check                     | `60s`          |
+| `services[].method`                    | Request method                                                  | `GET`          |
+| `services[].graphql`                   | Whether to wrap the body in a query param (`{"query":"$body"}`) | `false`        |
+| `services[].body`                      | Request body                                                    | `""`           |
+| `services[].headers`                   | Request headers                                                 | `{}`           |
+| `services[].alerts[].type`             | Type of alert. Valid types: `slack`, `twilio`, `custom`         | Required `""`  |
+| `services[].alerts[].enabled`          | Whether to enable the alert                                     | `false`        |
+| `services[].alerts[].threshold`        | Number of failures in a row needed before triggering the alert  | `3`            |
+| `services[].alerts[].description`      | Description of the alert. Will be included in the alert sent    | `""`           |
+| `services[].alerts[].send-on-resolved` | Whether to send a notification once a triggered alert subsides  | `false`        |
+| `alerting`                             | Configuration for alerting                                      | `{}`           |
+| `alerting.slack`                       | Webhook to use for alerts of type `slack`                       | `""`           |
+| `alerting.twilio`                      | Settings for alerts of type `twilio`                            | `""`           |
+| `alerting.twilio.sid`                  | Twilio account SID                                              | Required `""`  |
+| `alerting.twilio.token`                | Twilio auth token                                               | Required `""`  |
+| `alerting.twilio.from`                 | Number to send Twilio alerts from                               | Required `""`  |
+| `alerting.twilio.to`                   | Number to send twilio alerts to                                 | Required `""`  |
+| `alerting.custom`                      | Configuration for custom actions on failure or alerts           | `""`           |
+| `alerting.custom.url`                  | Custom alerting request url                                     | `""`           |
+| `alerting.custom.body`                 | Custom alerting request body.                                   | `""`           |
+| `alerting.custom.headers`              | Custom alerting request headers                                 | `{}`           |
+
+
+### Conditions
+
+Here are some examples of conditions you can use:
+
+| Condition                    | Description                                             | Passing values           | Failing values |
+| -----------------------------| ------------------------------------------------------- | ------------------------ | -------------- |
+| `[STATUS] == 200`            | Status must be equal to 200                             | 200                      | 201, 404, ...  |
+| `[STATUS] < 300`             | Status must lower than 300                              | 200, 201, 299            | 301, 302, ...  |
+| `[STATUS] <= 299`            | Status must be less than or equal to 299                | 200, 201, 299            | 301, 302, ...  |
+| `[STATUS] > 400`             | Status must be greater than 400                         | 401, 402, 403, 404       | 400, 200, ...  |
+| `[RESPONSE_TIME] < 500`      | Response time must be below 500ms                       | 100ms, 200ms, 300ms      | 500ms, 501ms   |
+| `[BODY] == 1`                | The body must be equal to 1                             | 1                        | Anything else  |
+| `[BODY].data.id == 1`        | The jsonpath `$.data.id` is equal to 1                  | `{"data":{"id":1}}`      |  |
+| `[BODY].data[0].id == 1`     | The jsonpath `$.data[0].id` is equal to 1               | `{"data":[{"id":1}]}`    |  |
+| `len([BODY].data) > 0`       | Array at jsonpath `$.data` has less than 5 elements     | `{"data":[{"id":1}]}`    |  |
+| `len([BODY].name) == 8`      | String at jsonpath `$.name` has a length of 8           | `{"name":"john.doe"}`    | `{"name":"bob"}` |
 
 
 ## Docker
 
-Building the Docker image is done as following:
+Building the Docker image is done as follows:
 
 ```
 docker build . -t gatus
@@ -56,3 +146,136 @@ go test ./... -mod vendor
 ## Using in Production
 
 See the [example](example) folder.
+
+
+## FAQ
+
+### Sending a GraphQL request
+
+By setting `services[].graphql` to true, the body will automatically be wrapped by the standard GraphQL `query` parameter.
+
+For instance, the following configuration:
+```yaml
+services:
+  - name: filter users by gender
+    url: http://localhost:8080/playground
+    method: POST
+    graphql: true
+    body: |
+      {
+        user(gender: "female") {
+          id
+          name
+          gender
+          avatar
+        }
+      }
+    headers:
+      Content-Type: application/json
+    conditions:
+      - "[STATUS] == 200"
+      - "[BODY].data.user[0].gender == female"
+```
+
+will send a `POST` request to `http://localhost:8080/playground` with the following body:
+```json
+{"query":"      {\n        user(gender: \"female\") {\n          id\n          name\n          gender\n          avatar\n        }\n      }"}
+```
+
+
+### Configuring Slack alerts
+
+```yaml
+alerting:
+  slack: "https://hooks.slack.com/services/**********/**********/**********"
+services:
+  - name: twinnation
+    interval: 30s
+    url: "https://twinnation.org/health"
+    alerts:
+      - type: slack
+        enabled: true
+        description: "healthcheck failed 3 times in a row"
+        send-on-resolved: true
+      - type: slack
+        enabled: true
+        threshold: 5
+        description: "healthcheck failed 5 times in a row"
+        send-on-resolved: true
+    conditions:
+      - "[STATUS] == 200"
+      - "[BODY].status == UP"
+      - "[RESPONSE_TIME] < 300"
+```
+
+Here's an example of what the notifications look like:
+
+![Slack notifications](.github/assets/slack-alerts.png)
+
+
+### Configuring Twilio alerts
+
+```yaml
+alerting:
+  twilio:
+    sid: "..."
+    token: "..."
+    from: "+1-234-567-8901"
+    to: "+1-234-567-8901"
+services:
+  - name: twinnation
+    interval: 30s
+    url: "https://twinnation.org/health"
+    alerts:
+      - type: twilio
+        enabled: true
+        threshold: 5
+        description: "healthcheck failed 5 times in a row"
+    conditions:
+      - "[STATUS] == 200"
+      - "[BODY].status == UP"
+      - "[RESPONSE_TIME] < 300"
+```
+
+
+### Configuring custom alerts
+
+While they're called alerts, you can use this feature to call anything. 
+
+For instance, you could automate rollbacks by having an application that keeps tracks of new deployments, and by 
+leveraging Gatus, you could have Gatus call that application endpoint when a service starts failing. Your application
+would then check if the service that started failing was recently deployed, and if it was, then automatically 
+roll it back.
+
+The values `[ALERT_DESCRIPTION]` and `[SERVICE_NAME]` are automatically substituted for the alert description and the 
+service name respectively in the body (`alerting.custom.body`) as well as the url (`alerting.custom.url`).
+
+If you have `send-on-resolved` set to `true`, you may want to use `[ALERT_TRIGGERED_OR_RESOLVED]` to differentiate
+the notifications. It will be replaced for either `TRIGGERED` or `RESOLVED`, based on the situation.
+
+For all intents and purpose, we'll configure the custom alert with a Slack webhook, but you can call anything you want.
+
+```yaml
+alerting:
+  custom:
+    url: "https://hooks.slack.com/services/**********/**********/**********"
+    method: "POST"
+    body: |
+      {
+        "text": "[ALERT_TRIGGERED_OR_RESOLVED]: [SERVICE_NAME] - [ALERT_DESCRIPTION]"
+      }
+services:
+  - name: twinnation
+    interval: 30s
+    url: "https://twinnation.org/health"
+    alerts:
+      - type: custom
+        enabled: true
+        threshold: 10
+        send-on-resolved: true
+        description: "healthcheck failed 10 times in a row"
+    conditions:
+      - "[STATUS] == 200"
+      - "[BODY].status == UP"
+      - "[RESPONSE_TIME] < 300"
+```

client/client.go

@@ -0,0 +1,19 @@
+package client
+
+import (
+	"net/http"
+	"time"
+)
+
+var (
+	client *http.Client
+)
+
+func GetHttpClient() *http.Client {
+	if client == nil {
+		client = &http.Client{
+			Timeout: time.Second * 10,
+		}
+	}
+	return client
+}

config.yaml

@@ -1,12 +1,17 @@
 metrics: true
 services:
-  - name: Twinnation
-    url: https://twinnation.org/health
+  - name: twinnation
     interval: 30s
+    url: https://twinnation.org/health
     conditions:
       - "[STATUS] == 200"
-  - name: GitHub API
-    url: https://api.github.com/healthz
+      - "[BODY].status == UP"
+      - "[RESPONSE_TIME] < 1000"
+  - name: twinnation-articles-api
     interval: 30s
+    url: "https://twinnation.org/api/v1/articles/24"
     conditions:
-      - "[STATUS] == 200"
+      - "[STATUS] == 200"
+      - "[BODY].id == 24"
+      - "[BODY].tags[0] == spring"
+      - "len([BODY].tags) > 0"

config/config.go

@@ -7,13 +7,11 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
-	"time"
 )
 
-type Config struct {
-	Metrics  bool            `yaml:"metrics"`
-	Services []*core.Service `yaml:"services"`
-}
+const (
+	DefaultConfigurationFilePath = "config/config.yaml"
+)
 
 var (
 	ErrNoServiceInConfig  = errors.New("configuration file should contain at least 1 service")
@@ -22,6 +20,13 @@ var (
 	config                *Config
 )
 
+type Config struct {
+	Metrics  bool                 `yaml:"metrics"`
+	Debug    bool                 `yaml:"debug"`
+	Alerting *core.AlertingConfig `yaml:"alerting"`
+	Services []*core.Service      `yaml:"services"`
+}
+
 func Get() *Config {
 	if config == nil {
 		panic(ErrConfigNotLoaded)
@@ -44,7 +49,7 @@ func Load(configFile string) error {
 }
 
 func LoadDefaultConfiguration() error {
-	err := Load("config/config.yaml")
+	err := Load(DefaultConfigurationFilePath)
 	if err != nil {
 		if err == ErrConfigFileNotFound {
 			return Load("config/config.yml")
@@ -74,9 +79,7 @@ func parseAndValidateConfigBytes(yamlBytes []byte) (config *Config, err error) {
 	} else {
 		// Set the default values if they aren't set
 		for _, service := range config.Services {
-			if service.Interval == 0 {
-				service.Interval = 10 * time.Second
-			}
+			service.Validate()
 		}
 	}
 	return

config/config_test.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"fmt"
+	"github.com/TwinProduction/gatus/core"
 	"testing"
 	"time"
 )
@@ -23,6 +24,9 @@ services:
 	if err != nil {
 		t.Error("No error should've been returned")
 	}
+	if config == nil {
+		t.Fatal("Config shouldn't have been nil")
+	}
 	if len(config.Services) != 2 {
 		t.Error("Should have returned two services")
 	}
@@ -36,8 +40,8 @@ services:
 	if config.Services[0].Interval != 15*time.Second {
 		t.Errorf("Interval should have been %s", 15*time.Second)
 	}
-	if config.Services[1].Interval != 10*time.Second {
-		t.Errorf("Interval should have been %s, because it is the default value", 10*time.Second)
+	if config.Services[1].Interval != 60*time.Second {
+		t.Errorf("Interval should have been %s, because it is the default value", 60*time.Second)
 	}
 	if len(config.Services[0].Conditions) != 1 {
 		t.Errorf("There should have been %d conditions", 1)
@@ -58,14 +62,17 @@ services:
 	if err != nil {
 		t.Error("No error should've been returned")
 	}
+	if config == nil {
+		t.Fatal("Config shouldn't have been nil")
+	}
 	if config.Metrics {
 		t.Error("Metrics should've been false by default")
 	}
 	if config.Services[0].Url != "https://twinnation.org/actuator/health" {
 		t.Errorf("URL should have been %s", "https://twinnation.org/actuator/health")
 	}
-	if config.Services[0].Interval != 10*time.Second {
-		t.Errorf("Interval should have been %s, because it is the default value", 10*time.Second)
+	if config.Services[0].Interval != 60*time.Second {
+		t.Errorf("Interval should have been %s, because it is the default value", 60*time.Second)
 	}
 }
 
@@ -81,14 +88,17 @@ services:
 	if err != nil {
 		t.Error("No error should've been returned")
 	}
+	if config == nil {
+		t.Fatal("Config shouldn't have been nil")
+	}
 	if !config.Metrics {
 		t.Error("Metrics should have been true")
 	}
 	if config.Services[0].Url != "https://twinnation.org/actuator/health" {
 		t.Errorf("URL should have been %s", "https://twinnation.org/actuator/health")
 	}
-	if config.Services[0].Interval != 10*time.Second {
-		t.Errorf("Interval should have been %s, because it is the default value", 10*time.Second)
+	if config.Services[0].Interval != 60*time.Second {
+		t.Errorf("Interval should have been %s, because it is the default value", 60*time.Second)
 	}
 }
 
@@ -107,3 +117,62 @@ badconfig:
 		t.Error("The error returned should have been of type ErrNoServiceInConfig")
 	}
 }
+
+func TestParseAndValidateConfigBytesWithAlerting(t *testing.T) {
+	config, err := parseAndValidateConfigBytes([]byte(`
+alerting:
+  slack: "http://example.com"
+services:
+  - name: twinnation
+    url: https://twinnation.org/actuator/health
+    alerts:
+      - type: slack
+        enabled: true
+        threshold: 7
+        description: "Healthcheck failed 7 times in a row"
+    conditions:
+      - "[STATUS] == 200"
+`))
+	if err != nil {
+		t.Error("No error should've been returned")
+	}
+	if config == nil {
+		t.Fatal("Config shouldn't have been nil")
+	}
+	if config.Metrics {
+		t.Error("Metrics should've been false by default")
+	}
+	if config.Alerting == nil {
+		t.Fatal("config.AlertingConfig shouldn't have been nil")
+	}
+	if config.Alerting.Slack != "http://example.com" {
+		t.Errorf("Slack webhook should've been %s, but was %s", "http://example.com", config.Alerting.Slack)
+	}
+	if len(config.Services) != 1 {
+		t.Error("There should've been 1 service")
+	}
+	if config.Services[0].Url != "https://twinnation.org/actuator/health" {
+		t.Errorf("URL should have been %s", "https://twinnation.org/actuator/health")
+	}
+	if config.Services[0].Interval != 60*time.Second {
+		t.Errorf("Interval should have been %s, because it is the default value", 60*time.Second)
+	}
+	if config.Services[0].Alerts == nil {
+		t.Fatal("The service alerts shouldn't have been nil")
+	}
+	if len(config.Services[0].Alerts) != 1 {
+		t.Fatal("There should've been 1 alert configured")
+	}
+	if !config.Services[0].Alerts[0].Enabled {
+		t.Error("The alert should've been enabled")
+	}
+	if config.Services[0].Alerts[0].Threshold != 7 {
+		t.Errorf("The threshold of the alert should've been %d, but it was %d", 7, config.Services[0].Alerts[0].Threshold)
+	}
+	if config.Services[0].Alerts[0].Type != core.SlackAlert {
+		t.Errorf("The type of the alert should've been %s, but it was %s", core.SlackAlert, config.Services[0].Alerts[0].Type)
+	}
+	if config.Services[0].Alerts[0].Description != "Healthcheck failed 7 times in a row" {
+		t.Errorf("The type of the alert should've been %s, but it was %s", "Healthcheck failed 7 times in a row", config.Services[0].Alerts[0].Description)
+	}
+}

core/alert.go

@@ -0,0 +1,27 @@
+package core
+
+// Alert is the service's alert configuration
+type Alert struct {
+	// Type of alert
+	Type AlertType `yaml:"type"`
+
+	// Enabled defines whether or not the alert is enabled
+	Enabled bool `yaml:"enabled"`
+
+	// Threshold is the number of failures in a row needed before triggering the alert
+	Threshold int `yaml:"threshold"`
+
+	// Description of the alert. Will be included in the alert sent.
+	Description string `yaml:"description"`
+
+	// SendOnResolved defines whether to send a second notification when the issue has been resolved
+	SendOnResolved bool `yaml:"send-on-resolved"`
+}
+
+type AlertType string
+
+const (
+	SlackAlert  AlertType = "slack"
+	TwilioAlert AlertType = "twilio"
+	CustomAlert AlertType = "custom"
+)

core/alerting.go

@@ -0,0 +1,149 @@
+package core
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"github.com/TwinProduction/gatus/client"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+type AlertingConfig struct {
+	Slack  string               `yaml:"slack"`
+	Twilio *TwilioAlertProvider `yaml:"twilio"`
+	Custom *CustomAlertProvider `yaml:"custom"`
+}
+
+type TwilioAlertProvider struct {
+	SID   string `yaml:"sid"`
+	Token string `yaml:"token"`
+	From  string `yaml:"from"`
+	To    string `yaml:"to"`
+}
+
+func (provider *TwilioAlertProvider) IsValid() bool {
+	return len(provider.Token) > 0 && len(provider.SID) > 0 && len(provider.From) > 0 && len(provider.To) > 0
+}
+
+type CustomAlertProvider struct {
+	Url     string            `yaml:"url"`
+	Method  string            `yaml:"method,omitempty"`
+	Body    string            `yaml:"body,omitempty"`
+	Headers map[string]string `yaml:"headers,omitempty"`
+}
+
+func (provider *CustomAlertProvider) IsValid() bool {
+	return len(provider.Url) > 0
+}
+
+func (provider *CustomAlertProvider) buildRequest(serviceName, alertDescription string, resolved bool) *http.Request {
+	body := provider.Body
+	providerUrl := provider.Url
+	if strings.Contains(body, "[ALERT_DESCRIPTION]") {
+		body = strings.ReplaceAll(body, "[ALERT_DESCRIPTION]", alertDescription)
+	}
+	if strings.Contains(body, "[SERVICE_NAME]") {
+		body = strings.ReplaceAll(body, "[SERVICE_NAME]", serviceName)
+	}
+	if strings.Contains(body, "[ALERT_TRIGGERED_OR_RESOLVED]") {
+		if resolved {
+			body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED")
+		} else {
+			body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED")
+		}
+	}
+	if strings.Contains(providerUrl, "[ALERT_DESCRIPTION]") {
+		providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_DESCRIPTION]", alertDescription)
+	}
+	if strings.Contains(providerUrl, "[SERVICE_NAME]") {
+		providerUrl = strings.ReplaceAll(providerUrl, "[SERVICE_NAME]", serviceName)
+	}
+	if strings.Contains(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]") {
+		if resolved {
+			providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED")
+		} else {
+			providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED")
+		}
+	}
+	bodyBuffer := bytes.NewBuffer([]byte(body))
+	request, _ := http.NewRequest(provider.Method, providerUrl, bodyBuffer)
+	for k, v := range provider.Headers {
+		request.Header.Set(k, v)
+	}
+	return request
+}
+
+func (provider *CustomAlertProvider) Send(serviceName, alertDescription string, resolved bool) error {
+	request := provider.buildRequest(serviceName, alertDescription, resolved)
+	response, err := client.GetHttpClient().Do(request)
+	if err != nil {
+		return err
+	}
+	if response.StatusCode > 399 {
+		return fmt.Errorf("call to provider alert returned status code %d", response.StatusCode)
+	}
+	return nil
+}
+
+func CreateSlackCustomAlertProvider(slackWebHookUrl string, service *Service, alert *Alert, result *Result, resolved bool) *CustomAlertProvider {
+	var message string
+	var color string
+	if resolved {
+		message = fmt.Sprintf("An alert for *%s* has been resolved after %d failures in a row", service.Name, service.NumberOfFailuresInARow)
+		color = "#36A64F"
+	} else {
+		message = fmt.Sprintf("An alert for *%s* has been triggered", service.Name)
+		color = "#DD0000"
+	}
+	var results string
+	for _, conditionResult := range result.ConditionResults {
+		var prefix string
+		if conditionResult.Success {
+			prefix = ":heavy_check_mark:"
+		} else {
+			prefix = ":x:"
+		}
+		results += fmt.Sprintf("%s - `%s`\n", prefix, conditionResult.Condition)
+	}
+	return &CustomAlertProvider{
+		Url:    slackWebHookUrl,
+		Method: "POST",
+		Body: fmt.Sprintf(`{
+  "text": "",
+  "attachments": [
+    {
+      "title": ":helmet_with_white_cross: Gatus",
+      "text": "%s:\n> %s",
+      "short": false,
+      "color": "%s",
+      "fields": [
+        {
+          "title": "Condition results",
+          "value": "%s",
+          "short": false
+        }
+      ]
+    },
+  ]
+}`, message, alert.Description, color, results),
+		Headers: map[string]string{"Content-Type": "application/json"},
+	}
+}
+
+func CreateTwilioCustomAlertProvider(provider *TwilioAlertProvider, message string) *CustomAlertProvider {
+	return &CustomAlertProvider{
+		Url:    fmt.Sprintf("https://api.twilio.com/2010-04-01/Accounts/%s/Messages.json", provider.SID),
+		Method: "POST",
+		Body: url.Values{
+			"To":   {provider.To},
+			"From": {provider.From},
+			"Body": {message},
+		}.Encode(),
+		Headers: map[string]string{
+			"Content-Type":  "application/x-www-form-urlencoded",
+			"Authorization": fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", provider.SID, provider.Token)))),
+		},
+	}
+}

core/condition.go

@@ -0,0 +1,51 @@
+package core
+
+import (
+	"fmt"
+	"log"
+	"strings"
+)
+
+type Condition string
+
+func (c *Condition) evaluate(result *Result) bool {
+	condition := string(*c)
+	success := false
+	var resolvedCondition string
+	if strings.Contains(condition, "==") {
+		parts := sanitizeAndResolve(strings.Split(condition, "=="), result)
+		success = parts[0] == parts[1]
+		resolvedCondition = fmt.Sprintf("%v == %v", parts[0], parts[1])
+	} else if strings.Contains(condition, "!=") {
+		parts := sanitizeAndResolve(strings.Split(condition, "!="), result)
+		success = parts[0] != parts[1]
+		resolvedCondition = fmt.Sprintf("%v != %v", parts[0], parts[1])
+	} else if strings.Contains(condition, "<=") {
+		parts := sanitizeAndResolveNumerical(strings.Split(condition, "<="), result)
+		success = parts[0] <= parts[1]
+		resolvedCondition = fmt.Sprintf("%v <= %v", parts[0], parts[1])
+	} else if strings.Contains(condition, ">=") {
+		parts := sanitizeAndResolveNumerical(strings.Split(condition, ">="), result)
+		success = parts[0] >= parts[1]
+		resolvedCondition = fmt.Sprintf("%v >= %v", parts[0], parts[1])
+	} else if strings.Contains(condition, ">") {
+		parts := sanitizeAndResolveNumerical(strings.Split(condition, ">"), result)
+		success = parts[0] > parts[1]
+		resolvedCondition = fmt.Sprintf("%v > %v", parts[0], parts[1])
+	} else if strings.Contains(condition, "<") {
+		parts := sanitizeAndResolveNumerical(strings.Split(condition, "<"), result)
+		success = parts[0] < parts[1]
+		resolvedCondition = fmt.Sprintf("%v < %v", parts[0], parts[1])
+	} else {
+		result.Errors = append(result.Errors, fmt.Sprintf("invalid condition '%s' has been provided", condition))
+		return false
+	}
+	conditionToDisplay := condition
+	// If the condition isn't a success, return what the resolved condition was too
+	if !success {
+		log.Printf("[Condition][evaluate] Condition '%s' did not succeed because '%s' is false", condition, resolvedCondition)
+		conditionToDisplay = fmt.Sprintf("%s (%s)", condition, resolvedCondition)
+	}
+	result.ConditionResults = append(result.ConditionResults, &ConditionResult{Condition: conditionToDisplay, Success: success})
+	return success
+}

core/condition_test.go

@@ -0,0 +1,186 @@
+package core
+
+import (
+	"testing"
+	"time"
+)
+
+func TestCondition_evaluateWithIp(t *testing.T) {
+	condition := Condition("[IP] == 127.0.0.1")
+	result := &Result{Ip: "127.0.0.1"}
+	condition.evaluate(result)
+	if !result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a success", condition)
+	}
+}
+
+func TestCondition_evaluateWithStatus(t *testing.T) {
+	condition := Condition("[STATUS] == 201")
+	result := &Result{HttpStatus: 201}
+	condition.evaluate(result)
+	if !result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a success", condition)
+	}
+}
+
+func TestCondition_evaluateWithStatusFailure(t *testing.T) {
+	condition := Condition("[STATUS] == 200")
+	result := &Result{HttpStatus: 500}
+	condition.evaluate(result)
+	if result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a failure", condition)
+	}
+}
+
+func TestCondition_evaluateWithStatusUsingLessThan(t *testing.T) {
+	condition := Condition("[STATUS] < 300")
+	result := &Result{HttpStatus: 201}
+	condition.evaluate(result)
+	if !result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a success", condition)
+	}
+}
+
+func TestCondition_evaluateWithStatusFailureUsingLessThan(t *testing.T) {
+	condition := Condition("[STATUS] < 300")
+	result := &Result{HttpStatus: 404}
+	condition.evaluate(result)
+	if result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a failure", condition)
+	}
+}
+
+func TestCondition_evaluateWithResponseTimeUsingLessThan(t *testing.T) {
+	condition := Condition("[RESPONSE_TIME] < 500")
+	result := &Result{Duration: time.Millisecond * 50}
+	condition.evaluate(result)
+	if !result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a success", condition)
+	}
+}
+
+func TestCondition_evaluateWithResponseTimeUsingGreaterThan(t *testing.T) {
+	condition := Condition("[RESPONSE_TIME] > 500")
+	result := &Result{Duration: time.Millisecond * 750}
+	condition.evaluate(result)
+	if !result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a success", condition)
+	}
+}
+
+func TestCondition_evaluateWithResponseTimeUsingGreaterThanOrEqualTo(t *testing.T) {
+	condition := Condition("[RESPONSE_TIME] >= 500")
+	result := &Result{Duration: time.Millisecond * 500}
+	condition.evaluate(result)
+	if !result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a success", condition)
+	}
+}
+
+func TestCondition_evaluateWithResponseTimeUsingLessThanOrEqualTo(t *testing.T) {
+	condition := Condition("[RESPONSE_TIME] <= 500")
+	result := &Result{Duration: time.Millisecond * 500}
+	condition.evaluate(result)
+	if !result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a success", condition)
+	}
+}
+
+func TestCondition_evaluateWithBody(t *testing.T) {
+	condition := Condition("[BODY] == test")
+	result := &Result{Body: []byte("test")}
+	condition.evaluate(result)
+	if !result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a success", condition)
+	}
+}
+
+func TestCondition_evaluateWithBodyJsonPath(t *testing.T) {
+	condition := Condition("[BODY].status == UP")
+	result := &Result{Body: []byte("{\"status\":\"UP\"}")}
+	condition.evaluate(result)
+	if !result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a success", condition)
+	}
+}
+
+func TestCondition_evaluateWithBodyJsonPathComplex(t *testing.T) {
+	condition := Condition("[BODY].data.name == john")
+	result := &Result{Body: []byte("{\"data\": {\"id\": 1, \"name\": \"john\"}}")}
+	condition.evaluate(result)
+	if !result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a success", condition)
+	}
+}
+
+func TestCondition_evaluateWithBodyJsonPathLongInt(t *testing.T) {
+	condition := Condition("[BODY].data.id == 1")
+	result := &Result{Body: []byte("{\"data\": {\"id\": 1}}")}
+	condition.evaluate(result)
+	if !result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a success", condition)
+	}
+}
+
+func TestCondition_evaluateWithBodyJsonPathComplexInt(t *testing.T) {
+	condition := Condition("[BODY].data[1].id == 2")
+	result := &Result{Body: []byte("{\"data\": [{\"id\": 1}, {\"id\": 2}, {\"id\": 3}]}")}
+	condition.evaluate(result)
+	if !result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a success", condition)
+	}
+}
+
+func TestCondition_evaluateWithBodyJsonPathComplexIntUsingGreaterThan(t *testing.T) {
+	condition := Condition("[BODY].data.id > 0")
+	result := &Result{Body: []byte("{\"data\": {\"id\": 1}}")}
+	condition.evaluate(result)
+	if !result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a success", condition)
+	}
+}
+
+func TestCondition_evaluateWithBodyJsonPathComplexIntFailureUsingGreaterThan(t *testing.T) {
+	condition := Condition("[BODY].data.id > 5")
+	result := &Result{Body: []byte("{\"data\": {\"id\": 1}}")}
+	condition.evaluate(result)
+	if result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a failure", condition)
+	}
+}
+
+func TestCondition_evaluateWithBodyJsonPathComplexIntUsingLessThan(t *testing.T) {
+	condition := Condition("[BODY].data.id < 5")
+	result := &Result{Body: []byte("{\"data\": {\"id\": 2}}")}
+	condition.evaluate(result)
+	if !result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a success", condition)
+	}
+}
+
+func TestCondition_evaluateWithBodyJsonPathComplexIntFailureUsingLessThan(t *testing.T) {
+	condition := Condition("[BODY].data.id < 5")
+	result := &Result{Body: []byte("{\"data\": {\"id\": 10}}")}
+	condition.evaluate(result)
+	if result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a failure", condition)
+	}
+}
+
+func TestCondition_evaluateWithBodySliceLength(t *testing.T) {
+	condition := Condition("len([BODY].data) == 3")
+	result := &Result{Body: []byte("{\"data\": [{\"id\": 1}, {\"id\": 2}, {\"id\": 3}]}")}
+	condition.evaluate(result)
+	if !result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a success", condition)
+	}
+}
+
+func TestCondition_evaluateWithBodyStringLength(t *testing.T) {
+	condition := Condition("len([BODY].name) == 8")
+	result := &Result{Body: []byte("{\"name\": \"john.doe\"}")}
+	condition.evaluate(result)
+	if !result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a success", condition)
+	}
+}

core/service.go

@@ -0,0 +1,162 @@
+package core
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"github.com/TwinProduction/gatus/client"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+var (
+	ErrNoCondition = errors.New("you must specify at least one condition per service")
+	ErrNoUrl       = errors.New("you must specify an url for each service")
+)
+
+// Service is the configuration of a monitored endpoint
+type Service struct {
+	// Name of the service. Can be anything.
+	Name string `yaml:"name"`
+
+	// URL to send the request to
+	Url string `yaml:"url"`
+
+	// Method of the request made to the url of the service
+	Method string `yaml:"method,omitempty"`
+
+	// Body of the request
+	Body string `yaml:"body,omitempty"`
+
+	// GraphQL is whether to wrap the body in a query param ({"query":"$body"})
+	GraphQL bool `yaml:"graphql,omitempty"`
+
+	// Headers of the request
+	Headers map[string]string `yaml:"headers,omitempty"`
+
+	// Interval is the duration to wait between every status check
+	Interval time.Duration `yaml:"interval,omitempty"`
+
+	// Conditions used to determine the health of the service
+	Conditions []*Condition `yaml:"conditions"`
+
+	// Alerts is the alerting configuration for the service in case of failure
+	Alerts []*Alert `yaml:"alerts"`
+
+	NumberOfFailuresInARow int
+}
+
+func (service *Service) Validate() {
+	// Set default values
+	if service.Interval == 0 {
+		service.Interval = 1 * time.Minute
+	}
+	if len(service.Method) == 0 {
+		service.Method = http.MethodGet
+	}
+	if len(service.Headers) == 0 {
+		service.Headers = make(map[string]string)
+	}
+	for _, alert := range service.Alerts {
+		if alert.Threshold <= 0 {
+			alert.Threshold = 3
+		}
+	}
+	if len(service.Url) == 0 {
+		panic(ErrNoUrl)
+	}
+	if len(service.Conditions) == 0 {
+		panic(ErrNoCondition)
+	}
+
+	// Make sure that the request can be created
+	_, err := http.NewRequest(service.Method, service.Url, bytes.NewBuffer([]byte(service.Body)))
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (service *Service) EvaluateConditions() *Result {
+	result := &Result{Success: true, Errors: []string{}}
+	service.getIp(result)
+	if len(result.Errors) == 0 {
+		service.call(result)
+	} else {
+		result.Success = false
+	}
+	for _, condition := range service.Conditions {
+		success := condition.evaluate(result)
+		if !success {
+			result.Success = false
+		}
+	}
+	result.Timestamp = time.Now()
+	return result
+}
+
+func (service *Service) GetAlertsTriggered() []Alert {
+	var alerts []Alert
+	if service.NumberOfFailuresInARow == 0 {
+		return alerts
+	}
+	for _, alert := range service.Alerts {
+		if alert.Enabled && alert.Threshold == service.NumberOfFailuresInARow {
+			alerts = append(alerts, *alert)
+			continue
+		}
+	}
+	return alerts
+}
+
+func (service *Service) getIp(result *Result) {
+	urlObject, err := url.Parse(service.Url)
+	if err != nil {
+		result.Errors = append(result.Errors, err.Error())
+		return
+	}
+	result.Hostname = urlObject.Hostname()
+	ips, err := net.LookupIP(urlObject.Hostname())
+	if err != nil {
+		result.Errors = append(result.Errors, err.Error())
+		return
+	}
+	result.Ip = ips[0].String()
+}
+
+func (service *Service) call(result *Result) {
+	request := service.buildRequest()
+	startTime := time.Now()
+	response, err := client.GetHttpClient().Do(request)
+	if err != nil {
+		result.Duration = time.Since(startTime)
+		result.Errors = append(result.Errors, err.Error())
+		return
+	}
+	result.Duration = time.Since(startTime)
+	result.HttpStatus = response.StatusCode
+	result.Body, err = ioutil.ReadAll(response.Body)
+	if err != nil {
+		result.Errors = append(result.Errors, err.Error())
+	}
+}
+
+func (service *Service) buildRequest() *http.Request {
+	var bodyBuffer *bytes.Buffer
+	if service.GraphQL {
+		graphQlBody := map[string]string{
+			"query": service.Body,
+		}
+		body, _ := json.Marshal(graphQlBody)
+		bodyBuffer = bytes.NewBuffer(body)
+	} else {
+		bodyBuffer = bytes.NewBuffer([]byte(service.Body))
+	}
+	request, _ := http.NewRequest(service.Method, service.Url, bodyBuffer)
+	for k, v := range service.Headers {
+		request.Header.Set(k, v)
+	}
+	return request
+}

core/service_test.go

@@ -0,0 +1,37 @@
+package core
+
+import (
+	"testing"
+)
+
+func TestIntegrationEvaluateConditions(t *testing.T) {
+	condition := Condition("[STATUS] == 200")
+	service := Service{
+		Name:       "TwiNNatioN",
+		Url:        "https://twinnation.org/health",
+		Conditions: []*Condition{&condition},
+	}
+	result := service.EvaluateConditions()
+	if !result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a success", condition)
+	}
+	if !result.Success {
+		t.Error("Because all conditions passed, this should have been a success")
+	}
+}
+
+func TestIntegrationEvaluateConditionsWithFailure(t *testing.T) {
+	condition := Condition("[STATUS] == 500")
+	service := Service{
+		Name:       "TwiNNatioN",
+		Url:        "https://twinnation.org/health",
+		Conditions: []*Condition{&condition},
+	}
+	result := service.EvaluateConditions()
+	if result.ConditionResults[0].Success {
+		t.Errorf("Condition '%s' should have been a failure", condition)
+	}
+	if result.Success {
+		t.Error("Because one of the conditions failed, success should have been false")
+	}
+}

core/types.go

@@ -1,12 +1,6 @@
 package core
 
 import (
-	"fmt"
-	"net"
-	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
 	"time"
 )
 
@@ -15,13 +9,9 @@ type HealthStatus struct {
 	Message string `json:"message,omitempty"`
 }
 
-type ServerMessage struct {
-	Error   bool   `json:"error"`
-	Message string `json:"message"`
-}
-
 type Result struct {
 	HttpStatus       int                `json:"status"`
+	Body             []byte             `json:"-"`
 	Hostname         string             `json:"hostname"`
 	Ip               string             `json:"-"`
 	Duration         time.Duration      `json:"duration"`
@@ -31,122 +21,7 @@ type Result struct {
 	Timestamp        time.Time          `json:"timestamp"`
 }
 
-type Service struct {
-	Name       string        `yaml:"name"`
-	Url        string        `yaml:"url"`
-	Interval   time.Duration `yaml:"interval,omitempty"`
-	Conditions []*Condition  `yaml:"conditions"`
-}
-
-func (service *Service) getIp(result *Result) {
-	urlObject, err := url.Parse(service.Url)
-	if err != nil {
-		result.Errors = append(result.Errors, err.Error())
-		return
-	}
-	result.Hostname = urlObject.Hostname()
-	ips, err := net.LookupIP(urlObject.Hostname())
-	if err != nil {
-		result.Errors = append(result.Errors, err.Error())
-		return
-	}
-	result.Ip = ips[0].String()
-}
-
-func (service *Service) getStatus(result *Result) {
-	client := &http.Client{
-		Timeout: time.Second * 10,
-	}
-	startTime := time.Now()
-	response, err := client.Get(service.Url)
-	if err != nil {
-		result.Errors = append(result.Errors, err.Error())
-		return
-	}
-	result.Duration = time.Now().Sub(startTime)
-	result.HttpStatus = response.StatusCode
-}
-
-func (service *Service) EvaluateConditions() *Result {
-	result := &Result{Success: true, Errors: []string{}}
-	service.getIp(result)
-	if len(result.Errors) == 0 {
-		service.getStatus(result)
-	} else {
-		result.Success = false
-	}
-	for _, condition := range service.Conditions {
-		success := condition.evaluate(result)
-		if !success {
-			result.Success = false
-		}
-	}
-	result.Timestamp = time.Now()
-	return result
-}
-
 type ConditionResult struct {
-	Condition   *Condition `json:"condition"`
-	Success     bool       `json:"success"`
-	Explanation string     `json:"-"`
-}
-
-type Condition string
-
-func (c *Condition) evaluate(result *Result) bool {
-	condition := string(*c)
-	if strings.Contains(condition, "==") {
-		parts := sanitizeAndResolve(strings.Split(condition, "=="), result)
-		if parts[0] == parts[1] {
-			result.ConditionResults = append(result.ConditionResults, &ConditionResult{
-				Condition:   c,
-				Success:     true,
-				Explanation: fmt.Sprintf("%s is equal to %s", parts[0], parts[1]),
-			})
-			return true
-		} else {
-			result.ConditionResults = append(result.ConditionResults, &ConditionResult{
-				Condition:   c,
-				Success:     false,
-				Explanation: fmt.Sprintf("%s is not equal to %s", parts[0], parts[1]),
-			})
-			return false
-		}
-	} else if strings.Contains(condition, "!=") {
-		parts := sanitizeAndResolve(strings.Split(condition, "!="), result)
-		if parts[0] != parts[1] {
-			result.ConditionResults = append(result.ConditionResults, &ConditionResult{
-				Condition:   c,
-				Success:     true,
-				Explanation: fmt.Sprintf("%s is not equal to %s", parts[0], parts[1]),
-			})
-			return true
-		} else {
-			result.ConditionResults = append(result.ConditionResults, &ConditionResult{
-				Condition:   c,
-				Success:     false,
-				Explanation: fmt.Sprintf("%s is equal to %s", parts[0], parts[1]),
-			})
-			return false
-		}
-	} else {
-		result.Errors = append(result.Errors, fmt.Sprintf("invalid condition '%s' has been provided", condition))
-		return false
-	}
-}
-
-func sanitizeAndResolve(list []string, result *Result) []string {
-	var sanitizedList []string
-	for _, element := range list {
-		element = strings.TrimSpace(element)
-		switch strings.ToUpper(element) {
-		case "[STATUS]":
-			element = strconv.Itoa(result.HttpStatus)
-		case "[IP]":
-			element = result.Ip
-		default:
-		}
-		sanitizedList = append(sanitizedList, element)
-	}
-	return sanitizedList
+	Condition string `json:"condition"`
+	Success   bool   `json:"success"`
 }

core/types_test.go

@@ -1,64 +0,0 @@
-package core
-
-import (
-	"testing"
-)
-
-func TestEvaluateWithIp(t *testing.T) {
-	condition := Condition("[IP] == 127.0.0.1")
-	result := &Result{Ip: "127.0.0.1"}
-	condition.evaluate(result)
-	if !result.ConditionResults[0].Success {
-		t.Errorf("Condition '%s' should have been a success", condition)
-	}
-}
-
-func TestEvaluateWithStatus(t *testing.T) {
-	condition := Condition("[STATUS] == 201")
-	result := &Result{HttpStatus: 201}
-	condition.evaluate(result)
-	if !result.ConditionResults[0].Success {
-		t.Errorf("Condition '%s' should have been a success", condition)
-	}
-}
-
-func TestEvaluateWithFailure(t *testing.T) {
-	condition := Condition("[STATUS] == 200")
-	result := &Result{HttpStatus: 500}
-	condition.evaluate(result)
-	if result.ConditionResults[0].Success {
-		t.Errorf("Condition '%s' should have been a failure", condition)
-	}
-}
-
-func TestIntegrationEvaluateConditions(t *testing.T) {
-	condition := Condition("[STATUS] == 200")
-	service := Service{
-		Name:       "GitHub",
-		Url:        "https://api.github.com/healthz",
-		Conditions: []*Condition{&condition},
-	}
-	result := service.EvaluateConditions()
-	if !result.ConditionResults[0].Success {
-		t.Errorf("Condition '%s' should have been a success", condition)
-	}
-	if !result.Success {
-		t.Error("Because all conditions passed, this should have been a success")
-	}
-}
-
-func TestIntegrationEvaluateConditionsWithFailure(t *testing.T) {
-	condition := Condition("[STATUS] == 500")
-	service := Service{
-		Name:       "GitHub",
-		Url:        "https://api.github.com/healthz",
-		Conditions: []*Condition{&condition},
-	}
-	result := service.EvaluateConditions()
-	if result.ConditionResults[0].Success {
-		t.Errorf("Condition '%s' should have been a failure", condition)
-	}
-	if result.Success {
-		t.Error("Because one of the conditions failed, success should have been false")
-	}
-}

core/util.go

@@ -0,0 +1,74 @@
+package core
+
+import (
+	"fmt"
+	"github.com/TwinProduction/gatus/jsonpath"
+	"strconv"
+	"strings"
+)
+
+const (
+	StatusPlaceholder       = "[STATUS]"
+	IPPlaceHolder           = "[IP]"
+	ResponseTimePlaceHolder = "[RESPONSE_TIME]"
+	BodyPlaceHolder         = "[BODY]"
+
+	LengthFunctionPrefix = "len("
+	FunctionSuffix       = ")"
+
+	InvalidConditionElementSuffix = "(INVALID)"
+)
+
+func sanitizeAndResolve(list []string, result *Result) []string {
+	var sanitizedList []string
+	body := strings.TrimSpace(string(result.Body))
+	for _, element := range list {
+		element = strings.TrimSpace(element)
+		switch strings.ToUpper(element) {
+		case StatusPlaceholder:
+			element = strconv.Itoa(result.HttpStatus)
+		case IPPlaceHolder:
+			element = result.Ip
+		case ResponseTimePlaceHolder:
+			element = strconv.Itoa(int(result.Duration.Milliseconds()))
+		case BodyPlaceHolder:
+			element = body
+		default:
+			// if starts with BodyPlaceHolder, then evaluate json path
+			if strings.Contains(element, BodyPlaceHolder) {
+				wantLength := false
+				if strings.HasPrefix(element, LengthFunctionPrefix) && strings.HasSuffix(element, FunctionSuffix) {
+					wantLength = true
+					element = strings.TrimSuffix(strings.TrimPrefix(element, LengthFunctionPrefix), FunctionSuffix)
+				}
+				resolvedElement, resolvedElementLength, err := jsonpath.Eval(strings.Replace(element, fmt.Sprintf("%s.", BodyPlaceHolder), "", 1), result.Body)
+				if err != nil {
+					result.Errors = append(result.Errors, err.Error())
+					element = fmt.Sprintf("%s %s", element, InvalidConditionElementSuffix)
+				} else {
+					if wantLength {
+						element = fmt.Sprintf("%d", resolvedElementLength)
+					} else {
+						element = resolvedElement
+					}
+				}
+			}
+		}
+		sanitizedList = append(sanitizedList, element)
+	}
+	return sanitizedList
+}
+
+func sanitizeAndResolveNumerical(list []string, result *Result) []int {
+	var sanitizedNumbers []int
+	sanitizedList := sanitizeAndResolve(list, result)
+	for _, element := range sanitizedList {
+		if number, err := strconv.Atoi(element); err != nil {
+			// Default to 0 if the string couldn't be converted to an integer
+			sanitizedNumbers = append(sanitizedNumbers, 0)
+		} else {
+			sanitizedNumbers = append(sanitizedNumbers, number)
+		}
+	}
+	return sanitizedNumbers
+}

example/docker-compose-grafana-prometheus/config.yaml

@@ -2,11 +2,12 @@ metrics: true
 services:
   - name: TwiNNatioN
     url: https://twinnation.org/health
-    interval: 10s
+    interval: 30s
     conditions:
       - "[STATUS] == 200"
   - name: GitHub
     url: https://api.github.com/healthz
+    interval: 5m
     conditions:
       - "[STATUS] == 200"
   - name: Example

example/kubernetes/gatus.yaml

@@ -10,6 +10,7 @@ data:
           - "[STATUS] == 200"
       - name: GitHub
         url: https://api.github.com/healthz
+        interval: 5m
         conditions:
           - "[STATUS] == 200"
       - name: Example

go.mod

@@ -1,6 +1,6 @@
 module github.com/TwinProduction/gatus
 
-go 1.13
+go 1.15
 
 require (
 	github.com/prometheus/client_golang v1.2.1

go.sum

@@ -18,6 +18,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=

jsonpath/jsonpath.go

@@ -0,0 +1,74 @@
+package jsonpath
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// Eval is a half-baked json path implementation that needs some love
+func Eval(path string, b []byte) (string, int, error) {
+	var object interface{}
+	err := json.Unmarshal(b, &object)
+	if err != nil {
+		// Try to unmarshal it into an array instead
+		return "", 0, err
+	}
+	return walk(path, object)
+}
+
+func walk(path string, object interface{}) (string, int, error) {
+	keys := strings.Split(path, ".")
+	currentKey := keys[0]
+	switch value := extractValue(currentKey, object).(type) {
+	case map[string]interface{}:
+		return walk(strings.Replace(path, fmt.Sprintf("%s.", currentKey), "", 1), value)
+	case string:
+		return value, len(value), nil
+	case []interface{}:
+		return fmt.Sprintf("%v", value), len(value), nil
+	case interface{}:
+		return fmt.Sprintf("%v", value), 1, nil
+	default:
+		return "", 0, fmt.Errorf("couldn't walk through '%s' because type was '%T', but expected 'map[string]interface{}'", currentKey, value)
+	}
+}
+
+func extractValue(currentKey string, value interface{}) interface{} {
+	// Check if the current key ends with [#]
+	if strings.HasSuffix(currentKey, "]") && strings.Contains(currentKey, "[") {
+		tmp := strings.SplitN(currentKey, "[", 3)
+		arrayIndex, err := strconv.Atoi(strings.Replace(tmp[1], "]", "", 1))
+		if err != nil {
+			return value
+		}
+		currentKey := tmp[0]
+		// if currentKey contains only an index (i.e. [0] or 0)
+		if len(currentKey) == 0 {
+			array := value.([]interface{})
+			if len(array) > arrayIndex {
+				if len(tmp) > 2 {
+					// Nested array? Go deeper.
+					return extractValue(fmt.Sprintf("%s[%s", currentKey, tmp[2]), array[arrayIndex])
+				}
+				return array[arrayIndex]
+			}
+			return nil
+		}
+		if value == nil || value.(map[string]interface{})[currentKey] == nil {
+			return nil
+		}
+		// if currentKey contains both a key and an index (i.e. data[0])
+		array := value.(map[string]interface{})[currentKey].([]interface{})
+		if len(array) > arrayIndex {
+			if len(tmp) > 2 {
+				// Nested array? Go deeper.
+				return extractValue(fmt.Sprintf("[%s", tmp[2]), array[arrayIndex])
+			}
+			return array[arrayIndex]
+		}
+		return nil
+	}
+	return value.(map[string]interface{})[currentKey]
+}

jsonpath/jsonpath_test.go

@@ -0,0 +1,152 @@
+package jsonpath
+
+import "testing"
+
+func TestEval(t *testing.T) {
+	path := "simple"
+	data := `{"simple": "value"}`
+
+	expectedOutput := "value"
+
+	output, outputLength, err := Eval(path, []byte(data))
+	if err != nil {
+		t.Error("Didn't expect any error, but got", err)
+	}
+	if outputLength != len(expectedOutput) {
+		t.Errorf("Expected output length to be %v, but was %v", len(expectedOutput), outputLength)
+	}
+	if output != expectedOutput {
+		t.Errorf("Expected output to be %v, but was %v", expectedOutput, output)
+	}
+}
+
+func TestEvalWithLongSimpleWalk(t *testing.T) {
+	path := "long.simple.walk"
+	data := `{"long": {"simple": {"walk": "value"}}}`
+
+	expectedOutput := "value"
+
+	output, _, err := Eval(path, []byte(data))
+	if err != nil {
+		t.Error("Didn't expect any error, but got", err)
+	}
+	if output != expectedOutput {
+		t.Errorf("Expected output to be %v, but was %v", expectedOutput, output)
+	}
+}
+
+func TestEvalWithArrayOfMaps(t *testing.T) {
+	path := "ids[1].id"
+	data := `{"ids": [{"id": 1}, {"id": 2}]}`
+
+	expectedOutput := "2"
+
+	output, _, err := Eval(path, []byte(data))
+	if err != nil {
+		t.Error("Didn't expect any error, but got", err)
+	}
+
+	if output != expectedOutput {
+		t.Errorf("Expected output to be %v, but was %v", expectedOutput, output)
+	}
+}
+
+func TestEvalWithArrayOfValues(t *testing.T) {
+	path := "ids[0]"
+	data := `{"ids": [1, 2]}`
+
+	expectedOutput := "1"
+
+	output, _, err := Eval(path, []byte(data))
+	if err != nil {
+		t.Error("Didn't expect any error, but got", err)
+	}
+	if output != expectedOutput {
+		t.Errorf("Expected output to be %v, but was %v", expectedOutput, output)
+	}
+}
+
+func TestEvalWithRootArrayOfValues(t *testing.T) {
+	path := "[1]"
+	data := `[1, 2]`
+
+	expectedOutput := "2"
+
+	output, _, err := Eval(path, []byte(data))
+	if err != nil {
+		t.Error("Didn't expect any error, but got", err)
+	}
+	if output != expectedOutput {
+		t.Errorf("Expected output to be %v, but was %v", expectedOutput, output)
+	}
+}
+
+func TestEvalWithRootArrayOfMaps(t *testing.T) {
+	path := "[0].id"
+	data := `[{"id": 1}, {"id": 2}]`
+
+	expectedOutput := "1"
+
+	output, _, err := Eval(path, []byte(data))
+	if err != nil {
+		t.Error("Didn't expect any error, but got", err)
+	}
+	if output != expectedOutput {
+		t.Errorf("Expected output to be %v, but was %v", expectedOutput, output)
+	}
+}
+
+func TestEvalWithRootArrayOfMapsUsingInvalidArrayIndex(t *testing.T) {
+	path := "[5].id"
+	data := `[{"id": 1}, {"id": 2}]`
+
+	_, _, err := Eval(path, []byte(data))
+	if err == nil {
+		t.Error("Should've returned an error, but didn't")
+	}
+}
+
+func TestEvalWithLongWalkAndArray(t *testing.T) {
+	path := "data.ids[0].id"
+	data := `{"data": {"ids": [{"id": 1}, {"id": 2}, {"id": 3}]}}`
+
+	expectedOutput := "1"
+
+	output, _, err := Eval(path, []byte(data))
+	if err != nil {
+		t.Error("Didn't expect any error, but got", err)
+	}
+	if output != expectedOutput {
+		t.Errorf("Expected output to be %v, but was %v", expectedOutput, output)
+	}
+}
+
+func TestEvalWithNestedArray(t *testing.T) {
+	path := "[3][2]"
+	data := `[[1, 2], [3, 4], [], [5, 6, 7]]`
+
+	expectedOutput := "7"
+
+	output, _, err := Eval(path, []byte(data))
+	if err != nil {
+		t.Error("Didn't expect any error, but got", err)
+	}
+	if output != expectedOutput {
+		t.Errorf("Expected output to be %v, but was %v", expectedOutput, output)
+	}
+}
+
+func TestEvalWithMapOfNestedArray(t *testing.T) {
+	path := "data[1][1]"
+	data := `{"data": [["a", "b", "c"], ["d", "e", "f"]]}`
+
+	expectedOutput := "e"
+
+	output, _, err := Eval(path, []byte(data))
+	if err != nil {
+		t.Error("Didn't expect any error, but got", err)
+	}
+	if output != expectedOutput {
+		t.Errorf("Expected output to be %v, but was %v", expectedOutput, output)
+	}
+}

main.go

@@ -1,19 +1,28 @@
 package main
 
 import (
-	"encoding/json"
+	"bytes"
+	"compress/gzip"
 	"github.com/TwinProduction/gatus/config"
-	"github.com/TwinProduction/gatus/core"
 	"github.com/TwinProduction/gatus/watchdog"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"log"
 	"net/http"
 	"os"
+	"strings"
+	"time"
+)
+
+const CacheTTL = 10 * time.Second
+
+var (
+	cachedServiceResults          []byte
+	cachedServiceResultsGzipped   []byte
+	cachedServiceResultsTimestamp time.Time
 )
 
 func main() {
 	cfg := loadConfiguration()
-	go watchdog.Monitor(cfg)
 	http.HandleFunc("/api/v1/results", serviceResultsHandler)
 	http.HandleFunc("/health", healthHandler)
 	http.Handle("/", http.FileServer(http.Dir("./static")))
@@ -21,6 +30,7 @@ func main() {
 		http.Handle("/metrics", promhttp.Handler())
 	}
 	log.Println("[main][main] Listening on port 8080")
+	go watchdog.Monitor(cfg)
 	log.Fatal(http.ListenAndServe(":8080", nil))
 }
 
@@ -38,23 +48,37 @@ func loadConfiguration() *config.Config {
 	return config.Get()
 }
 
-func serviceResultsHandler(writer http.ResponseWriter, _ *http.Request) {
-	serviceResults := watchdog.GetServiceResults()
+func serviceResultsHandler(writer http.ResponseWriter, r *http.Request) {
+	if isExpired := cachedServiceResultsTimestamp.IsZero() || time.Now().Sub(cachedServiceResultsTimestamp) > CacheTTL; isExpired {
+		buffer := &bytes.Buffer{}
+		gzipWriter := gzip.NewWriter(buffer)
+		data, err := watchdog.GetJsonEncodedServiceResults()
+		if err != nil {
+			log.Printf("[main][serviceResultsHandler] Unable to marshal object to JSON: %s", err.Error())
+			writer.WriteHeader(http.StatusInternalServerError)
+			_, _ = writer.Write([]byte("Unable to marshal object to JSON"))
+			return
+		}
+		gzipWriter.Write(data)
+		gzipWriter.Close()
+		cachedServiceResults = data
+		cachedServiceResultsGzipped = buffer.Bytes()
+		cachedServiceResultsTimestamp = time.Now()
+	}
+	var data []byte
+	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
+		writer.Header().Set("Content-Encoding", "gzip")
+		data = cachedServiceResultsGzipped
+	} else {
+		data = cachedServiceResults
+	}
 	writer.Header().Add("Content-type", "application/json")
 	writer.WriteHeader(http.StatusOK)
-	_, _ = writer.Write(structToJsonBytes(serviceResults))
+	_, _ = writer.Write(data)
 }
 
 func healthHandler(writer http.ResponseWriter, _ *http.Request) {
 	writer.Header().Add("Content-type", "application/json")
 	writer.WriteHeader(http.StatusOK)
-	_, _ = writer.Write(structToJsonBytes(&core.HealthStatus{Status: "UP"}))
-}
-
-func structToJsonBytes(obj interface{}) []byte {
-	bytes, err := json.Marshal(obj)
-	if err != nil {
-		log.Printf("[main][structToJsonBytes] Unable to marshall object to JSON: %s", err.Error())
-	}
-	return bytes
+	_, _ = writer.Write([]byte("{\"status\":\"UP\"}"))
 }

static/index.html

@@ -1,9 +1,16 @@
 <!DOCTYPE html>
 <html lang="en">
 <head>
-	<title>Status</title>
+	<title>Health Dashboard</title>
+	<meta name="viewport" content="width=device-width, initial-scale=1" />
 	<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
 	<style>
+		html, body {
+			background-color: #f7f9fb;
+		}
+		html {
+			height: 100%;
+		}
 		#results div.container:first-child {
 			border-top-left-radius: 3px;
 			border-top-right-radius: 3px;
@@ -22,92 +29,260 @@
 			margin-right: 4px;
 			background-color: #28a745;
 		}
+		.status {
+			cursor: pointer;
+			transition: all 500ms ease-in-out;
+			overflow-x: hidden;
+			padding: .25em 0;
+			color: white;
+		}
+		.title {
+			font-size: 2.5rem;
+		}
+		.status:hover {
+			opacity: 0.7;
+			transition: opacity 100ms ease-in-out;
+			color: black;
+		}
+		.status-over-time {
+			overflow: auto;
+		}
+		.status-over-time>span:not(:last-child) {
+			margin-left: 2px;
+		}
+		.status-time-ago {
+			color: #6a737d;
+			opacity: 0.5;
+			margin-top: 5px;
+		}
+		.status-min-max-ms {
+			overflow-x: hidden;
+		}
+		#tooltip {
+			position: fixed;
+			top: 0;
+			left: 0;
+			background-color: white;
+			border: 1px solid lightgray;
+			border-radius: 4px;
+			padding: 6px;
+			font-size: 13px;
+		}
+		#tooltip code {
+			color: #212529;
+			line-height: 1;
+		}
+		#tooltip .tooltip-title {
+			font-weight: bold;
+			margin-bottom: 0;
+			display: block;
+		}
+		#tooltip .tooltip-title {
+			margin-top: 8px;
+		}
+		#tooltip>.tooltip-title:first-child {
+			margin-top: 0;
+		}
 	</style>
 </head>
 <body>
-	<div class="container my-3 bg-light rounded p-4 border shadow">
-		<div class="text-center mb-3">
-			<div class="display-4">Status</div>
+	<div class="container my-3 rounded p-3 border shadow">
+		<div class="mb-2">
+			<div class="row">
+				<div class="col-8 text-left my-auto">
+					<div class="title display-4">Health Status</div>
+				</div>
+				<div class="col-4 text-right">
+					<img src="logo.png" alt="GaTuS" style="position: relative; min-width: 50px; max-width: 200px; width: 20%;"/>
+				</div>
+			</div>
 		</div>
-		<div id="results">
+		<div id="results"></div>
+	</div>
+
+	<div id="tooltip" style="display: none">
+		<div class="tooltip-title">Timestamp:</div>
+		<code id="tooltip-timestamp">...</code>
+		<div class="tooltip-title">Response time:</div>
+		<code id="tooltip-response-time">...</code>
+		<div class="tooltip-title">Conditions:</div>
+		<code id="tooltip-conditions">...</code>
+		<div id="tooltip-errors-container">
+			<div class="tooltip-title">Errors:</div>
+			<code id="tooltip-errors">...</code>
 		</div>
 	</div>
 
 	<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
-	<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js" integrity="sha384-UO2eT0CpHqdSJQ6hJty5KVphtPhzWj9WO1clHTMGa3JDZwrnQq4sF86dIHNDz0W1" crossorigin="anonymous"></script>
 	<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js" integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM" crossorigin="anonymous"></script>
 
 	<script>
-		//const OK = "<div class='status-ok' title='__RESPONSE_TIME____CONDITIONS____ERRORS__'></div>"
-		const OK = "<span class='badge badge-success ml-1' style='width: 5%' title='__RESPONSE_TIME____CONDITIONS____ERRORS__'>&#10003;</span>";
-		const NOK = "<span class='badge badge-danger ml-1' style='width: 5%' title='__RESPONSE_TIME____CONDITIONS____ERRORS__'>X</span>";
+		let serviceStatuses = {};
+		let timerHandler = 0;
+		let userClickedStatus = false;
 
-		function generateServiceResultBox(serviceResult) {
-			let output = (serviceResult.success ? OK : NOK);
-			output = output.replace("__RESPONSE_TIME__", "Response time:\n" + parseInt(serviceResult.duration/1000000) + "ms");
+		function showTooltip(serviceName, index, element) {
+			userClickedStatus = false;
+			clearTimeout(timerHandler);
+			let serviceResult = serviceStatuses[serviceName][index];
+			$("#tooltip-timestamp").text(prettifyTimestamp(serviceResult.timestamp));
+			$("#tooltip-response-time").text(parseInt(serviceResult.duration/1000000) + "ms");
+			// Populate the condition section
 			let conditions = "";
-			for (let conditionResultIndex in serviceResult['condition-results']) {
-				let conditionResult = serviceResult['condition-results'][conditionResultIndex];
-				conditions += "\n- " + (conditionResult.success ? "&#10003;" : "X") + " ~ " + conditionResult.condition;
+			for (let i in serviceResult['condition-results']) {
+				let conditionResult = serviceResult['condition-results'][i];
+				conditions += (conditionResult.success ? "&#10003;" : "X") + " ~ " + htmlEntities(conditionResult.condition) + "<br />";
 			}
-			output = output.replace("__CONDITIONS__", "\n\nConditions:" + conditions);
-			if (serviceResult['errors'].length > 0) {
+			$("#tooltip-conditions").html(conditions);
+			// Populate the error section only if there are errors
+			if (serviceResult.errors && serviceResult.errors.length > 0) {
 				let errors = "";
-				for (let errorIndex in serviceResult['errors']) {
-					errors += "\n- " + serviceResult['errors'][errorIndex];
+				for (let i in serviceResult.errors) {
+					errors += "- " + htmlEntities(serviceResult.errors[i]) + "<br />";
 				}
-				output = output.replace("__ERRORS__", "\n\nErrors: " + errors);
+				$("#tooltip-errors").html(errors);
+				$("#tooltip-errors-container").show();
 			} else {
-				output = output.replace("__ERRORS__", "");
+				$("#tooltip-errors-container").hide();
 			}
-			return output;
+			// Position tooltip
+			$("#tooltip").css({top: "0px", left: "0px"}).show();
+			let targetTopPosition = element.getBoundingClientRect().y + 30;
+			let targetLeftPosition = element.getBoundingClientRect().x;
+			// Make adjustments if necessary
+			let tooltipBoundingClientRect = document.querySelector('#tooltip').getBoundingClientRect();
+			if (targetLeftPosition + window.scrollX + tooltipBoundingClientRect.width + 50 > document.body.getBoundingClientRect().width) {
+				targetLeftPosition = element.getBoundingClientRect().x - tooltipBoundingClientRect.width + element.getBoundingClientRect().width;
+				if (targetLeftPosition < 0) {
+					targetLeftPosition += -targetLeftPosition;
+				}
+			}
+			if (targetTopPosition + window.scrollY + tooltipBoundingClientRect.height + 50 > document.body.getBoundingClientRect().height && targetTopPosition >= 0) {
+				targetTopPosition = element.getBoundingClientRect().y - (tooltipBoundingClientRect.height + 10);
+				if (targetTopPosition < 0) {
+					targetTopPosition = element.getBoundingClientRect().y + 30;
+				}
+			}
+			$("#tooltip").css({top: targetTopPosition + "px", left: targetLeftPosition + "px"});
 		}
-		
+
+		function fadeTooltip() {
+			if (!userClickedStatus) {
+				timerHandler = setTimeout(function () {
+					$("#tooltip").hide();
+				}, 500);
+			}
+		}
+
+		function createStatusBadge(serviceName, index, success) {
+			if (success) {
+				return "<span class='status badge badge-success' style='width: 5%' onmouseenter='showTooltip(\""+serviceName+"\", "+index+", this)' onmouseleave='fadeTooltip()' onclick='userClickedStatus = !userClickedStatus;'>&#10003;</span>";
+			}
+			return "<span class='status badge badge-danger' style='width: 5%' onmouseenter='showTooltip(\""+serviceName+"\", "+index+", this)' onmouseleave='fadeTooltip()' onclick='userClickedStatus = !userClickedStatus;'>X</span>";
+		}
+
 		function refreshResults() {
 			$.getJSON("/api/v1/results", function (data) {
-				let output = "";
-				for (let serviceName in data) {
-					let serviceStatusOverTime = "";
-					let hostname = data[serviceName][data[serviceName].length-1].hostname 
-					let minResponseTime = null;
-					let maxResponseTime = null;
-					for (let key in data[serviceName]) {
-						let serviceResult = data[serviceName][key];
-						console.log(serviceResult);
-						serviceStatusOverTime = generateServiceResultBox(serviceResult) + serviceStatusOverTime;
-						const responseTime = parseInt(serviceResult.duration/1000000);
-						if (minResponseTime == null || minResponseTime > responseTime) {
-							minResponseTime = responseTime;
-						}
-						if (maxResponseTime == null || maxResponseTime < responseTime) {
-							maxResponseTime = responseTime;
-						}
-					}
-					output += ""
-						+ "<div class='container p-2 border-left border-right border-top border-black'>"
-						+ "  <div class='row mb-2'>"
-						+ "    <div class='col-8'>"
-						+ "      <span class='font-weight-bold'>" + serviceName + "</span> <span class='text-secondary font-weight-lighter'>- " + hostname + "</span>"
-						+ "    </div>"
-						+ "    <div class='col-4 text-right'>"
-						+ "      <span class='font-weight-lighter'>" + (minResponseTime === maxResponseTime ? minResponseTime : (minResponseTime + "-" + maxResponseTime)) + "ms</span>"
-						+ "    </div>"
-						+ "  </div>"
-						+ "  <div class='row'>"
-						+ "    <div class='col-12 d-flex flex-row-reverse'>"
-						+ "      " + serviceStatusOverTime
-						+ "    </div>"
-						+ "  </div>"
-						+ "</div>";
+				// Update the table only if there's a change
+				if (JSON.stringify(serviceStatuses) !== JSON.stringify(data)) {
+					serviceStatuses = data;
+					buildTable();
 				}
-				$("#results").html(output);
 			});
 		}
 
+		function buildTable() {
+			let output = "";
+			for (let serviceName in serviceStatuses) {
+				let serviceStatusOverTime = "";
+				let hostname = serviceStatuses[serviceName][serviceStatuses[serviceName].length-1].hostname
+				let minResponseTime = null;
+				let maxResponseTime = null;
+				let newestTimestamp = null;
+				let oldestTimestamp = null;
+				for (let key in serviceStatuses[serviceName]) {
+					let serviceResult = serviceStatuses[serviceName][key];
+					serviceStatusOverTime = createStatusBadge(serviceName, key, serviceResult.success) + serviceStatusOverTime;
+					const responseTime = parseInt(serviceResult.duration/1000000);
+					if (minResponseTime == null || minResponseTime > responseTime) {
+						minResponseTime = responseTime;
+					}
+					if (maxResponseTime == null || maxResponseTime < responseTime) {
+						maxResponseTime = responseTime;
+					}
+					const timestamp = new Date(serviceResult.timestamp);
+					if (newestTimestamp == null || newestTimestamp < timestamp) {
+						newestTimestamp = timestamp;
+					}
+					if (oldestTimestamp == null || oldestTimestamp > timestamp) {
+						oldestTimestamp = timestamp;
+					}
+				}
+				output += ""
+					+ "<div class='container py-3 border-left border-right border-top border-black'>"
+					+ "  <div class='row mb-2'>"
+					+ "    <div class='col-md-10'>"
+					+ "      <span class='font-weight-bold'>" + serviceName + "</span> <span class='text-secondary font-weight-lighter'>- " + hostname + "</span>"
+					+ "    </div>"
+					+ "    <div class='col-md-2 text-right'>"
+					+ "      <span class='font-weight-lighter status-min-max-ms'>" + (minResponseTime === maxResponseTime ? minResponseTime : (minResponseTime + "-" + maxResponseTime)) + "ms</span>"
+					+ "    </div>"
+					+ "  </div>"
+					+ "  <div class='row'>"
+					+ "    <div class='col-12 d-flex flex-row-reverse status-over-time'>"
+					+ "      " + serviceStatusOverTime
+					+ "    </div>"
+					+ "  </div>"
+					+ "  <div class='row status-time-ago'>"
+					+ "    <div class='col-6'>"
+					+ "      " + generatePrettyTimeAgo(oldestTimestamp)
+					+ "    </div>"
+					+ "    <div class='col-6 text-right'>"
+					+ "      " + generatePrettyTimeAgo(newestTimestamp)
+					+ "    </div>"
+					+ "  </div>"
+					+ "</div>";
+			}
+			$("#results").html(output);
+		}
+
+		function prettifyTimestamp(timestamp) {
+			let date = new Date(timestamp);
+			let YYYY = date.getFullYear();
+			let MM = ((date.getMonth()+1)<10?"0":"")+""+(date.getMonth()+1);
+			let DD = ((date.getDate())<10?"0":"")+""+(date.getDate());
+			let hh = ((date.getHours())<10?"0":"")+""+(date.getHours());
+			let mm = ((date.getMinutes())<10?"0":"")+""+(date.getMinutes());
+			let ss = ((date.getSeconds())<10?"0":"")+""+(date.getSeconds());
+			return YYYY + "-" + MM + "-" + DD + " " + hh + ":" + mm + ":" + ss;
+		}
+
+		function generatePrettyTimeAgo(t) {
+			let differenceInMs = new Date().getTime() - new Date(t).getTime();
+			if (differenceInMs > 3600000) {
+				let hours = (differenceInMs/3600000).toFixed(0);
+				return hours + " hour" + (hours !== "1" ? "s" : "") + " ago";
+			}
+			if (differenceInMs > 60000) {
+				let minutes = (differenceInMs/60000).toFixed(0);
+				return minutes + " minute" + (minutes !== "1" ? "s" : "") + " ago";
+			}
+			return (differenceInMs/1000).toFixed(0) + " seconds ago";
+		}
+
+		function htmlEntities(s) {
+			return String(s)
+				.replace(/&/g, '&amp;')
+				.replace(/</g, '&lt;')
+				.replace(/>/g, '&gt;')
+				.replace(/"/g, '&quot;')
+				.replace(/'/g, '&apos;');
+		}
+
 		refreshResults();
 		setInterval(function() {
 			refreshResults();
-		}, 10000);
+		}, 30000);
 	</script>
 </body>
 </html>

vendor/modules.txt

@@ -7,6 +7,7 @@ github.com/golang/protobuf/proto
 # github.com/matttproud/golang_protobuf_extensions v1.0.1
 github.com/matttproud/golang_protobuf_extensions/pbutil
 # github.com/prometheus/client_golang v1.2.1
+## explicit
 github.com/prometheus/client_golang/prometheus
 github.com/prometheus/client_golang/prometheus/internal
 github.com/prometheus/client_golang/prometheus/promauto
@@ -24,4 +25,5 @@ github.com/prometheus/procfs/internal/util
 # golang.org/x/sys v0.0.0-20191010194322-b09406accb47
 golang.org/x/sys/windows
 # gopkg.in/yaml.v2 v2.2.2
+## explicit
 gopkg.in/yaml.v2

watchdog/watchdog.go

@@ -1,6 +1,8 @@
 package watchdog
 
 import (
+	"encoding/json"
+	"fmt"
 	"github.com/TwinProduction/gatus/config"
 	"github.com/TwinProduction/gatus/core"
 	"github.com/TwinProduction/gatus/metric"
@@ -11,41 +13,161 @@ import (
 
 var (
 	serviceResults = make(map[string][]*core.Result)
-	rwLock         sync.RWMutex
+
+	// serviceResultsMutex is used to prevent concurrent map access
+	serviceResultsMutex sync.RWMutex
+
+	// monitoringMutex is used to prevent multiple services from being evaluated at the same time.
+	// Without this, conditions using response time may become inaccurate.
+	monitoringMutex sync.Mutex
 )
 
-func GetServiceResults() *map[string][]*core.Result {
-	return &serviceResults
+// GetJsonEncodedServiceResults returns a list of the last 20 results for each services encoded using json.Marshal.
+// The reason why the encoding is done here is because we use a mutex to prevent concurrent map access.
+func GetJsonEncodedServiceResults() ([]byte, error) {
+	serviceResultsMutex.RLock()
+	data, err := json.Marshal(serviceResults)
+	serviceResultsMutex.RUnlock()
+	return data, err
 }
 
+// Monitor loops over each services and starts a goroutine to monitor each services separately
 func Monitor(cfg *config.Config) {
 	for _, service := range cfg.Services {
 		go monitor(service)
 		// To prevent multiple requests from running at the same time
-		time.Sleep(500 * time.Millisecond)
+		time.Sleep(1111 * time.Millisecond)
 	}
 }
 
+// monitor monitors a single service in a loop
 func monitor(service *core.Service) {
+	cfg := config.Get()
 	for {
 		// By placing the lock here, we prevent multiple services from being monitored at the exact same time, which
 		// could cause performance issues and return inaccurate results
-		rwLock.Lock()
-		log.Printf("[watchdog][Monitor] Monitoring serviceName=%s", service.Name)
+		monitoringMutex.Lock()
+		if cfg.Debug {
+			log.Printf("[watchdog][monitor] Monitoring serviceName=%s", service.Name)
+		}
 		result := service.EvaluateConditions()
 		metric.PublishMetricsForService(service, result)
+		serviceResultsMutex.Lock()
 		serviceResults[service.Name] = append(serviceResults[service.Name], result)
 		if len(serviceResults[service.Name]) > 20 {
 			serviceResults[service.Name] = serviceResults[service.Name][1:]
 		}
-		rwLock.Unlock()
+		serviceResultsMutex.Unlock()
+		var extra string
+		if !result.Success {
+			extra = fmt.Sprintf("responseBody=%s", result.Body)
+		}
 		log.Printf(
-			"[watchdog][Monitor] Finished monitoring serviceName=%s; errors=%d; requestDuration=%s",
+			"[watchdog][monitor] Monitored serviceName=%s; success=%v; errors=%d; requestDuration=%s; %s",
 			service.Name,
+			result.Success,
 			len(result.Errors),
 			result.Duration.Round(time.Millisecond),
+			extra,
 		)
-		log.Printf("[watchdog][Monitor] Waiting interval=%s before monitoring serviceName=%s", service.Interval, service.Name)
+		handleAlerting(service, result)
+		if cfg.Debug {
+			log.Printf("[watchdog][monitor] Waiting for interval=%s before monitoring serviceName=%s again", service.Interval, service.Name)
+		}
+		monitoringMutex.Unlock()
 		time.Sleep(service.Interval)
 	}
 }
+
+func handleAlerting(service *core.Service, result *core.Result) {
+	cfg := config.Get()
+	if cfg.Alerting == nil {
+		return
+	}
+	if result.Success {
+		if service.NumberOfFailuresInARow > 0 {
+			for _, alert := range service.Alerts {
+				if !alert.Enabled || !alert.SendOnResolved || alert.Threshold > service.NumberOfFailuresInARow {
+					continue
+				}
+				var alertProvider *core.CustomAlertProvider
+				if alert.Type == core.SlackAlert {
+					if len(cfg.Alerting.Slack) > 0 {
+						log.Printf("[watchdog][handleAlerting] Sending Slack alert because alert with description=%s has been resolved", alert.Description)
+						alertProvider = core.CreateSlackCustomAlertProvider(cfg.Alerting.Slack, service, alert, result, true)
+					} else {
+						log.Printf("[watchdog][handleAlerting] Not sending Slack alert despite being triggered, because there is no Slack webhook configured")
+					}
+				} else if alert.Type == core.TwilioAlert {
+					if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() {
+						log.Printf("[watchdog][handleAlerting] Sending Twilio alert because alert with description=%s has been resolved", alert.Description)
+						alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description))
+					} else {
+						log.Printf("[watchdog][handleAlerting] Not sending Twilio alert despite being resolved, because Twilio isn't configured properly")
+					}
+				} else if alert.Type == core.CustomAlert {
+					if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() {
+						log.Printf("[watchdog][handleAlerting] Sending custom alert because alert with description=%s has been resolved", alert.Description)
+						alertProvider = &core.CustomAlertProvider{
+							Url:     cfg.Alerting.Custom.Url,
+							Method:  cfg.Alerting.Custom.Method,
+							Body:    cfg.Alerting.Custom.Body,
+							Headers: cfg.Alerting.Custom.Headers,
+						}
+					} else {
+						log.Printf("[watchdog][handleAlerting] Not sending custom alert despite being resolved, because the custom provider isn't configured properly")
+					}
+				}
+				if alertProvider != nil {
+					err := alertProvider.Send(service.Name, alert.Description, true)
+					if err != nil {
+						log.Printf("[watchdog][handleAlerting] Ran into error sending an alert: %s", err.Error())
+					}
+				}
+			}
+		}
+		service.NumberOfFailuresInARow = 0
+	} else {
+		service.NumberOfFailuresInARow++
+		for _, alert := range service.Alerts {
+			// If the alert hasn't been triggered, move to the next one
+			if !alert.Enabled || alert.Threshold != service.NumberOfFailuresInARow {
+				continue
+			}
+			var alertProvider *core.CustomAlertProvider
+			if alert.Type == core.SlackAlert {
+				if len(cfg.Alerting.Slack) > 0 {
+					log.Printf("[watchdog][handleAlerting] Sending Slack alert because alert with description=%s has been triggered", alert.Description)
+					alertProvider = core.CreateSlackCustomAlertProvider(cfg.Alerting.Slack, service, alert, result, false)
+				} else {
+					log.Printf("[watchdog][handleAlerting] Not sending Slack alert despite being triggered, because there is no Slack webhook configured")
+				}
+			} else if alert.Type == core.TwilioAlert {
+				if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() {
+					log.Printf("[watchdog][handleAlerting] Sending Twilio alert because alert with description=%s has been triggered", alert.Description)
+					alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description))
+				} else {
+					log.Printf("[watchdog][handleAlerting] Not sending Twilio alert despite being triggered, because Twilio config settings missing")
+				}
+			} else if alert.Type == core.CustomAlert {
+				if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() {
+					log.Printf("[watchdog][handleAlerting] Sending custom alert because alert with description=%s has been triggered", alert.Description)
+					alertProvider = &core.CustomAlertProvider{
+						Url:     cfg.Alerting.Custom.Url,
+						Method:  cfg.Alerting.Custom.Method,
+						Body:    cfg.Alerting.Custom.Body,
+						Headers: cfg.Alerting.Custom.Headers,
+					}
+				} else {
+					log.Printf("[watchdog][handleAlerting] Not sending custom alert despite being triggered, because there is no custom url configured")
+				}
+			}
+			if alertProvider != nil {
+				err := alertProvider.Send(service.Name, alert.Description, false)
+				if err != nil {
+					log.Printf("[watchdog][handleAlerting] Ran into error sending an alert: %s", err.Error())
+				}
+			}
+		}
+	}
+}