Compare commits

...

40 Commits

Author SHA1 Message Date
TwinProduction
8fd6eddc16 Make sure that the SHA512 hash is lowercase 2020-10-14 21:45:45 -04:00
TwinProduction
402525d572 Update Kubernetes example 2020-10-14 20:07:56 -04:00
Chris C
68c8076c73 Merge pull request #20 from TwinProduction/basic-auth
Implement basic authentication for dashboard
2020-10-14 19:51:50 -04:00
TwinProduction
331b089afb Document security feature (#8) 2020-10-14 19:26:34 -04:00
TwinProduction
ea454eb3b2 Add tests for basic authentication (#8) 2020-10-14 19:26:14 -04:00
TwinProduction
3e9aba6cbb Wrap /api/v1/results handler with security handler if security config is valid (#8) 2020-10-14 19:25:50 -04:00
TwinProduction
3fb7d27f3a Add security handler (#8) 2020-10-14 19:24:36 -04:00
TwinProduction
c0b1fefec8 Support SHA512 as hashing algorithm (#8) 2020-10-14 19:24:00 -04:00
TwinProduction
9220a777bb Start working on #8: Support basic authentication for the dashboard 2020-10-14 19:22:58 -04:00
TwinProduction
70c9c4b87c Switch to MIT 2020-10-12 23:49:25 -04:00
TwinProduction
148b7bbdfd Add follow badge 2020-10-11 01:13:14 -04:00
TwinProduction
a556ca797b Add Go version badge 2020-10-07 19:14:21 -04:00
TwinProduction
3d7dbf4bba Improve documentation 2020-10-06 19:07:47 -04:00
TwinProduction
230c8821ec Merge util.go into condition.go 2020-10-04 20:00:24 -04:00
TwinProduction
ecef3d9538 Add result.Connected in integration tests 2020-10-04 19:55:19 -04:00
TwinProduction
3ecfe4d322 Close #18: Support monitoring TCP services 2020-10-04 19:49:02 -04:00
TwinProduction
6a3f65db7f Close #14: Support skipping certificate verification (services[].insecure) 2020-10-04 17:01:10 -04:00
Chris C
ed490669b1 Merge pull request #15 from hurnhu/refresh-interval
Close #1: Add support for refresh interval
2020-10-02 12:20:09 -04:00
TwinProduction
187b38ee22 Fix format 2020-10-02 12:10:09 -04:00
TwinProduction
261fe27b2e Move refresh interval to bottom left corner instead
+ Use icon
+ Minor improvements
2020-10-02 12:08:52 -04:00
mike
a29f175cba default 30 seconds 2020-10-02 10:10:48 -04:00
mike
f26dacaf22 changed interval input to a select. select is autopopulated with options on load. default is 30 seconds 2020-10-02 09:48:23 -04:00
TwinProduction
b4d81a9661 Tables are cuter 2020-10-01 20:04:04 -04:00
TwinProduction
81c59f02d8 Close #16: Update documentation 2020-10-01 19:57:52 -04:00
TwinProduction
8101646ba5 Work on #16: Support patterns 2020-10-01 19:57:11 -04:00
Michael LaPan
93b7220d5f added code to set the refresh interval 2020-10-01 10:45:20 -04:00
TwinProduction
18d3236586 Make resolved/triggered alert logs more obvious 2020-09-29 20:06:47 -04:00
TwinProduction
bc16ae1794 Add tests for comparing two placeholders 2020-09-29 19:33:06 -04:00
Chris C
d019942278 Update README.md 2020-09-28 12:50:22 -04:00
TwinProduction
8c5551f93b Update build.yml 2020-09-27 20:39:42 -04:00
TwinProduction
3b15feaf7d Update README.md 2020-09-26 15:21:06 -04:00
TwinProduction
326ad2c80d Update README.md 2020-09-26 15:18:52 -04:00
TwinProduction
b4b9f5c1be Improve documentation 2020-09-26 15:15:50 -04:00
TwinProduction
ac5ad9d173 Work on common provider interface to make adding new providers easier 2020-09-26 14:23:43 -04:00
TwinProduction
7dcd462883 Clean up old code 2020-09-25 18:23:30 -04:00
TwinProduction
5655661778 Add missing provider.go file 2020-09-24 19:54:15 -04:00
TwinProduction
fac7b8551a Start working on implementing common provider interface 2020-09-24 19:52:59 -04:00
TwinProduction
59fed008e0 Rename service's EvaluateConditions() to EvaluateHealth() 2020-09-24 19:49:32 -04:00
TwinProduction
286e8f8590 Add config validation logs on application start 2020-09-22 17:46:40 -04:00
TwinProduction
4daf261d95 Minor update 2020-09-21 10:41:23 -04:00
32 changed files with 995 additions and 423 deletions

View File

@@ -14,13 +14,13 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Set up Go 1.14
- name: Set up Go 1.15
uses: actions/setup-go@v1
with:
go-version: 1.14
go-version: 1.15
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v1
uses: actions/checkout@v2
- name: Build binary to make sure it works
run: go build -mod vendor
- name: Test

View File

@@ -1,201 +1,9 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
MIT License
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
Copyright (c) 2020 TwinProduction
1. Definitions.
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2020 TwinProduction
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

142
README.md
View File

@@ -1,13 +1,15 @@
![Gatus](static/logo-with-name.png)
![build](https://github.com/TwinProduction/gatus/workflows/build/badge.svg?branch=master)
[![Go Report Card](https://goreportcard.com/badge/github.com/TwinProduction/gatus)](https://goreportcard.com/report/github.com/TwinProduction/gatus)
[![Go Report Card](https://goreportcard.com/badge/github.com/TwinProduction/gatus?)](https://goreportcard.com/report/github.com/TwinProduction/gatus)
[![Follow TwinProduction](https://img.shields.io/github/followers/TwinProduction?label=Follow&style=social)](https://github.com/TwinProduction)
[![Go version](https://img.shields.io/github/go-mod/go-version/TwinProduction/gatus.svg)](https://github.com/TwinProduction/gatus)
[![Docker pulls](https://img.shields.io/docker/pulls/twinproduction/gatus.svg)](https://cloud.docker.com/repository/docker/twinproduction/gatus)
A service health dashboard in Go that is meant to be used as a docker
image with a custom configuration file.
I personally deploy it in my Kubernetes cluster and have it monitor the status of my
I personally deploy it in my Kubernetes cluster and let it monitor the status of my
core applications: https://status.twinnation.org/
@@ -17,6 +19,8 @@ core applications: https://status.twinnation.org/
- [Usage](#usage)
- [Configuration](#configuration)
- [Conditions](#conditions)
- [Placeholders](#placeholders)
- [Functions](#functions)
- [Alerting](#alerting)
- [Configuring Slack alerts](#configuring-slack-alerts)
- [Configuring PagerDuty alerts](#configuring-pagerduty-alerts)
@@ -27,6 +31,9 @@ core applications: https://status.twinnation.org/
- [Using in Production](#using-in-production)
- [FAQ](#faq)
- [Sending a GraphQL request](#sending-a-graphql-request)
- [Recommended interval](#recommended-interval)
- [Default timeouts](#default-timeouts)
- [Monitoring a TCP service](#monitoring-a-tcp-service)
## Features
@@ -75,13 +82,14 @@ Note that you can also add environment variables in the configuration file (i.e.
### Configuration
| Parameter | Description | Default |
| ---------------------------------------- | ----------------------------------------------------------------------------- | -------------- |
|:---------------------------------------- |:----------------------------------------------------------------------------- |:-------------- |
| `debug` | Whether to enable debug logs | `false` |
| `metrics` | Whether to expose metrics at /metrics | `false` |
| `services` | List of services to monitor | Required `[]` |
| `services[].name` | Name of the service. Can be anything. | Required `""` |
| `services[].url` | URL to send the request to | Required `""` |
| `services[].conditions` | Conditions used to determine the health of the service | `[]` |
| `services[].insecure` | Whether to skip verifying the server's certificate chain and host name | `false` |
| `services[].interval` | Duration to wait between every status check | `60s` |
| `services[].method` | Request method | `GET` |
| `services[].graphql` | Whether to wrap the body in a query param (`{"query":"$body"}`) | `false` |
@@ -94,19 +102,23 @@ Note that you can also add environment variables in the configuration file (i.e.
| `services[].alerts[].send-on-resolved` | Whether to send a notification once a triggered alert is marked as resolved | `false` |
| `services[].alerts[].description` | Description of the alert. Will be included in the alert sent | `""` |
| `alerting` | Configuration for alerting | `{}` |
| `alerting.slack` | Configuration for alerts of type `slack` | `""` |
| `alerting.slack` | Configuration for alerts of type `slack` | `{}` |
| `alerting.slack.webhook-url` | Slack Webhook URL | Required `""` |
| `alerting.pagerduty` | Configuration for alerts of type `pagerduty` | `""` |
| `alerting.pagerduty` | Configuration for alerts of type `pagerduty` | `{}` |
| `alerting.pagerduty.integration-key` | PagerDuty Events API v2 integration key. | Required `""` |
| `alerting.twilio` | Settings for alerts of type `twilio` | `""` |
| `alerting.twilio` | Settings for alerts of type `twilio` | `{}` |
| `alerting.twilio.sid` | Twilio account SID | Required `""` |
| `alerting.twilio.token` | Twilio auth token | Required `""` |
| `alerting.twilio.from` | Number to send Twilio alerts from | Required `""` |
| `alerting.twilio.to` | Number to send twilio alerts to | Required `""` |
| `alerting.custom` | Configuration for custom actions on failure or alerts | `""` |
| `alerting.custom` | Configuration for custom actions on failure or alerts | `{}` |
| `alerting.custom.url` | Custom alerting request url | Required `""` |
| `alerting.custom.body` | Custom alerting request body. | `""` |
| `alerting.custom.headers` | Custom alerting request headers | `{}` |
| `security` | Security configuration | `{}` |
| `security.basic` | Basic authentication security configuration | `{}` |
| `security.basic.username` | Username for Basic authentication | Required `""` |
| `security.basic.password-sha512` | Password's SHA512 hash for Basic authentication | Required `""` |
### Conditions
@@ -114,23 +126,48 @@ Note that you can also add environment variables in the configuration file (i.e.
Here are some examples of conditions you can use:
| Condition | Description | Passing values | Failing values |
| -----------------------------| ------------------------------------------------------- | -------------------------- | -------------- |
|:-----------------------------|:------------------------------------------------------- |:-------------------------- | -------------- |
| `[STATUS] == 200` | Status must be equal to 200 | 200 | 201, 404, ... |
| `[STATUS] < 300` | Status must lower than 300 | 200, 201, 299 | 301, 302, ... |
| `[STATUS] <= 299` | Status must be less than or equal to 299 | 200, 201, 299 | 301, 302, ... |
| `[STATUS] > 400` | Status must be greater than 400 | 401, 402, 403, 404 | 400, 200, ... |
| `[CONNECTED] == true` | Connection to host must've been successful | true, false | |
| `[RESPONSE_TIME] < 500` | Response time must be below 500ms | 100ms, 200ms, 300ms | 500ms, 501ms |
| `[BODY] == 1` | The body must be equal to 1 | 1 | Anything else |
| `[IP] == 127.0.0.1` | Target IP must be 127.0.0.1 | 127.0.0.1 | 0.0.0.0 |
| `[BODY] == 1` | The body must be equal to 1 | 1 | `{}`, `2`, ... |
| `[BODY].user.name == john` | JSONPath value of `$.user.name` is equal to `john` | `{"user":{"name":"john"}}` | |
| `[BODY].data[0].id == 1` | JSONPath value of `$.data[0].id` is equal to 1 | `{"data":[{"id":1}]}` | |
| `[BODY].age == [BODY].id` | JSONPath value of `$.age` is equal JSONPath `$.id` | `{"age":1,"id":1}` | |
| `len([BODY].data) < 5` | Array at JSONPath `$.data` has less than 5 elements | `{"data":[{"id":1}]}` | |
| `len([BODY].name) == 8` | String at JSONPath `$.name` has a length of 8 | `{"name":"john.doe"}` | `{"name":"bob"}` |
| `[BODY].name == pat(john*)` | String at JSONPath `$.name` matches pattern `john*` | `{"name":"john.doe"}` | `{"name":"bob"}` |
#### Placeholders
| Placeholder | Description | Example of resolved value |
|:----------------------- |:------------------------------------------------------- |:------------------------- |
| `[STATUS]` | Resolves into the HTTP status of the request | 404
| `[RESPONSE_TIME]` | Resolves into the response time the request took, in ms | 10
| `[IP]` | Resolves into the IP of the target host | 192.168.0.232
| `[BODY]` | Resolves into the response body. Supports JSONPath. | `{"name":"john.doe"}`
| `[CONNECTED]` | Resolves into whether a connection could be established | `true`
#### Functions
| Function | Description | Example |
|:-----------|:---------------------------------------------------------------------------------------------------------------- |:-------------------------- |
| `len` | Returns the length of the object/slice. Works only with the `[BODY]` placeholder. | `len([BODY].username) > 8`
| `pat` | Specifies that the string passed as parameter should be evaluated as a pattern. Works only with `==` and `!=`. | `[IP] == pat(192.168.*)`
**NOTE**: Use `pat` only when you need to. `[STATUS] == pat(2*)` is a lot more expensive than `[STATUS] < 300`.
### Alerting
#### Configuring Slack alerts
```yaml
@@ -139,8 +176,8 @@ alerting:
webhook-url: "https://hooks.slack.com/services/**********/**********/**********"
services:
- name: twinnation
interval: 30s
url: "https://twinnation.org/health"
interval: 30s
alerts:
- type: slack
enabled: true
@@ -175,15 +212,15 @@ alerting:
integration-key: "********************************"
services:
- name: twinnation
interval: 30s
url: "https://twinnation.org/health"
interval: 30s
alerts:
- type: pagerduty
enabled: true
failure-threshold: 3
success-threshold: 5
description: "healthcheck failed 3 times in a row"
send-on-resolved: true
description: "healthcheck failed 3 times in a row"
conditions:
- "[STATUS] == 200"
- "[BODY].status == UP"
@@ -208,8 +245,8 @@ services:
- type: twilio
enabled: true
failure-threshold: 5
description: "healthcheck failed 5 times in a row"
send-on-resolved: true
description: "healthcheck failed 5 times in a row"
conditions:
- "[STATUS] == 200"
- "[BODY].status == UP"
@@ -245,8 +282,8 @@ alerting:
}
services:
- name: twinnation
interval: 30s
url: "https://twinnation.org/health"
interval: 30s
alerts:
- type: custom
enabled: true
@@ -267,12 +304,12 @@ Other than using one of the examples provided in the `examples` folder, you can
creating a configuration file - we'll call it `config.yaml` for this example - and running the following
command:
```
docker run -p 8080:8080 --mount type=bind,source="$(pwd)"/test.yaml,target=/config/config.yaml --name gatus twinproduction/gatus
docker run -p 8080:8080 --mount type=bind,source="$(pwd)"/config.yaml,target=/config/config.yaml --name gatus twinproduction/gatus
```
If you're on Windows, replace `"$(pwd)"` by the absolute path to your current directory, e.g.:
```
docker run -p 8080:8080 --mount type=bind,source=E:/Go/src/github.com/TwinProduction/gatus/test.yaml,target=/config/config.yaml --name gatus twinproduction/gatus
docker run -p 8080:8080 --mount type=bind,source=C:/Users/Chris/Desktop/config.yaml,target=/config/config.yaml --name gatus twinproduction/gatus
```
## Running the tests
@@ -320,3 +357,74 @@ will send a `POST` request to `http://localhost:8080/playground` with the follow
```json
{"query":" {\n user(gender: \"female\") {\n id\n name\n gender\n avatar\n }\n }"}
```
### Recommended interval
To ensure that Gatus provides reliable and accurate results (i.e. response time), Gatus only evaluates one service at a time.
In other words, even if you have multiple services with the exact same interval, they will not execute at the same time.
You can test this yourself by running Gatus with several services configured with a very short, unrealistic interval,
such as 1ms. You'll notice that the response time does not fluctuate - that is because while services are evaluated on
different goroutines, there's a global lock that prevents multiple services from running at the same time.
Unfortunately, there is a drawback. If you have a lot of services, including some that are very slow or prone to time out (the default
time out is 10s for HTTP and 5s for TCP), then it means that for the entire duration of the request, no other services can be evaluated.
**This does mean that Gatus will be unable to evaluate the health of other services**.
The interval does not include the duration of the request itself, which means that if a service has an interval of 30s
and the request takes 2s to complete, the timestamp between two evaluations will be 32s, not 30s.
While this does not prevent Gatus' from performing health checks on all other services, it may cause Gatus to be unable
to respect the configured interval, for instance:
- Service A has an interval of 5s, and times out after 10s to complete
- Service B has an interval of 5s, and takes 1ms to complete
- Service B will be unable to run every 5s, because service A's health evaluation takes longer than its interval
To sum it up, while Gatus can really handle any interval you throw at it, you're better off having slow requests with
higher interval.
As a rule of the thumb, I personally set interval for more complex health checks to `5m` (5 minutes) and
simple health checks used for alerting (PagerDuty/Twilio) to `30s`.
### Default timeouts
| Protocol | Timeout |
|:-------- |:------- |
| HTTP | 10s
| TCP | 5s
### Monitoring a TCP service
By prefixing `services[].url` with `tcp:\\`, you can monitor TCP services at a very basic level:
```yaml
- name: redis
url: "tcp://127.0.0.1:6379"
interval: 30s
conditions:
- "[CONNECTED] == true"
```
Placeholders `[STATUS]` and `[BODY]` as well as the fields `services[].body`, `services[].insecure`,
`services[].headers`, `services[].method` and `services[].graphql` are not supported for TCP services.
**NOTE**: `[CONNECTED] == true` does not guarantee that the service itself is healthy - it only guarantees that there's
something at the given address listening to the given port, and that a connection to that address was successfully
established.
### Basic authentication
You can require Basic authentication by leveraging the `security.basic` configuration:
```yaml
security:
basic:
username: "john.doe"
password-sha512: "6b97ed68d14eb3f1aa959ce5d49c7dc612e1eb1dafd73b1e705847483fd6a6c809f2ceb4e8df6ff9984c6298ff0285cace6614bf8daa9f0070101b6c89899e22"
```
The example above will require that you authenticate with the username `john.doe` as well as the password `hunter2`.

View File

@@ -4,11 +4,14 @@ import (
"bytes"
"fmt"
"github.com/TwinProduction/gatus/client"
"github.com/TwinProduction/gatus/core"
"io/ioutil"
"net/http"
"strings"
)
// AlertProvider is the configuration necessary for sending an alert using a custom HTTP request
// Technically, all alert providers should be reachable using the custom alert provider
type AlertProvider struct {
Url string `yaml:"url"`
Method string `yaml:"method,omitempty"`
@@ -16,10 +19,16 @@ type AlertProvider struct {
Headers map[string]string `yaml:"headers,omitempty"`
}
// IsValid returns whether the provider's configuration is valid
func (provider *AlertProvider) IsValid() bool {
return len(provider.Url) > 0
}
// ToCustomAlertProvider converts the provider into a custom.AlertProvider
func (provider *AlertProvider) ToCustomAlertProvider(service *core.Service, alert *core.Alert, result *core.Result, resolved bool) *AlertProvider {
return provider
}
func (provider *AlertProvider) buildRequest(serviceName, alertDescription string, resolved bool) *http.Request {
body := provider.Body
providerUrl := provider.Url
@@ -64,7 +73,7 @@ func (provider *AlertProvider) buildRequest(serviceName, alertDescription string
// Send a request to the alert provider and return the body
func (provider *AlertProvider) Send(serviceName, alertDescription string, resolved bool) ([]byte, error) {
request := provider.buildRequest(serviceName, alertDescription, resolved)
response, err := client.GetHttpClient().Do(request)
response, err := client.GetHttpClient(false).Do(request)
if err != nil {
return nil, err
}

View File

@@ -6,16 +6,28 @@ import (
"github.com/TwinProduction/gatus/core"
)
// AlertProvider is the configuration necessary for sending an alert using PagerDuty
type AlertProvider struct {
IntegrationKey string `yaml:"integration-key"`
}
// IsValid returns whether the provider's configuration is valid
func (provider *AlertProvider) IsValid() bool {
return len(provider.IntegrationKey) == 32
}
// https://developer.pagerduty.com/docs/events-api-v2/trigger-events/
func (provider *AlertProvider) ToCustomAlertProvider(eventAction, resolveKey string, service *core.Service, message string) *custom.AlertProvider {
func (provider *AlertProvider) ToCustomAlertProvider(service *core.Service, alert *core.Alert, result *core.Result, resolved bool) *custom.AlertProvider {
var message, eventAction, resolveKey string
if resolved {
message = fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description)
eventAction = "resolve"
resolveKey = alert.ResolveKey
} else {
message = fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description)
eventAction = "trigger"
resolveKey = ""
}
return &custom.AlertProvider{
Url: "https://events.pagerduty.com/v2/enqueue",
Method: "POST",

View File

@@ -0,0 +1,15 @@
package provider
import (
"github.com/TwinProduction/gatus/alerting/provider/custom"
"github.com/TwinProduction/gatus/core"
)
// AlertProvider is the interface that each providers should implement
type AlertProvider interface {
// IsValid returns whether the provider's configuration is valid
IsValid() bool
// ToCustomAlertProvider converts the provider into a custom.AlertProvider
ToCustomAlertProvider(service *core.Service, alert *core.Alert, result *core.Result, resolved bool) *custom.AlertProvider
}

View File

@@ -6,14 +6,17 @@ import (
"github.com/TwinProduction/gatus/core"
)
// AlertProvider is the configuration necessary for sending an alert using Slack
type AlertProvider struct {
WebhookUrl string `yaml:"webhook-url"`
}
// IsValid returns whether the provider's configuration is valid
func (provider *AlertProvider) IsValid() bool {
return len(provider.WebhookUrl) > 0
}
// ToCustomAlertProvider converts the provider into a custom.AlertProvider
func (provider *AlertProvider) ToCustomAlertProvider(service *core.Service, alert *core.Alert, result *core.Result, resolved bool) *custom.AlertProvider {
var message string
var color string

View File

@@ -4,9 +4,11 @@ import (
"encoding/base64"
"fmt"
"github.com/TwinProduction/gatus/alerting/provider/custom"
"github.com/TwinProduction/gatus/core"
"net/url"
)
// AlertProvider is the configuration necessary for sending an alert using Twilio
type AlertProvider struct {
SID string `yaml:"sid"`
Token string `yaml:"token"`
@@ -14,11 +16,19 @@ type AlertProvider struct {
To string `yaml:"to"`
}
// IsValid returns whether the provider's configuration is valid
func (provider *AlertProvider) IsValid() bool {
return len(provider.Token) > 0 && len(provider.SID) > 0 && len(provider.From) > 0 && len(provider.To) > 0
}
func (provider *AlertProvider) ToCustomAlertProvider(message string) *custom.AlertProvider {
// ToCustomAlertProvider converts the provider into a custom.AlertProvider
func (provider *AlertProvider) ToCustomAlertProvider(service *core.Service, alert *core.Alert, result *core.Result, resolved bool) *custom.AlertProvider {
var message string
if resolved {
message = fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description)
} else {
message = fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description)
}
return &custom.AlertProvider{
Url: fmt.Sprintf("https://api.twilio.com/2010-04-01/Accounts/%s/Messages.json", provider.SID),
Method: "POST",

View File

@@ -1,19 +1,45 @@
package client
import (
"crypto/tls"
"net"
"net/http"
"time"
)
var (
client *http.Client
secureHttpClient *http.Client
insecureHttpClient *http.Client
)
func GetHttpClient() *http.Client {
if client == nil {
client = &http.Client{
Timeout: time.Second * 10,
func GetHttpClient(insecure bool) *http.Client {
if insecure {
if insecureHttpClient == nil {
insecureHttpClient = &http.Client{
Timeout: time.Second * 10,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
},
}
}
return insecureHttpClient
} else {
if secureHttpClient == nil {
secureHttpClient = &http.Client{
Timeout: time.Second * 10,
}
}
return secureHttpClient
}
return client
}
func CanCreateConnectionToTcpService(address string) bool {
conn, err := net.DialTimeout("tcp", address, 5*time.Second)
if err != nil {
return false
}
_ = conn.Close()
return true
}

28
client/client_test.go Normal file
View File

@@ -0,0 +1,28 @@
package client
import (
"testing"
)
func TestGetHttpClient(t *testing.T) {
if secureHttpClient != nil {
t.Error("secureHttpClient should've been nil since it hasn't been called a single time yet")
}
if insecureHttpClient != nil {
t.Error("insecureHttpClient should've been nil since it hasn't been called a single time yet")
}
_ = GetHttpClient(false)
if secureHttpClient == nil {
t.Error("secureHttpClient shouldn't have been nil, since it has been called once")
}
if insecureHttpClient != nil {
t.Error("insecureHttpClient should've been nil since it hasn't been called a single time yet")
}
_ = GetHttpClient(true)
if secureHttpClient == nil {
t.Error("secureHttpClient shouldn't have been nil, since it has been called once")
}
if insecureHttpClient == nil {
t.Error("insecureHttpClient shouldn't have been nil, since it has been called once")
}
}

View File

@@ -1,15 +1,15 @@
metrics: true
services:
- name: twinnation
url: "https://twinnation.org/health"
interval: 30s
url: https://twinnation.org/health
conditions:
- "[STATUS] == 200"
- "[BODY].status == UP"
- "[RESPONSE_TIME] < 1000"
- name: cat-fact
interval: 1m
url: "https://cat-fact.herokuapp.com/facts/random"
interval: 1m
conditions:
- "[STATUS] == 200"
- "[BODY].deleted == false"

View File

@@ -3,7 +3,9 @@ package config
import (
"errors"
"github.com/TwinProduction/gatus/alerting"
"github.com/TwinProduction/gatus/alerting/provider"
"github.com/TwinProduction/gatus/core"
"github.com/TwinProduction/gatus/security"
"gopkg.in/yaml.v2"
"io/ioutil"
"log"
@@ -11,19 +13,24 @@ import (
)
const (
// DefaultConfigurationFilePath is the default path that will be used to search for the configuration file
// if a custom path isn't configured through the GATUS_CONFIG_FILE environment variable
DefaultConfigurationFilePath = "config/config.yaml"
)
var (
ErrNoServiceInConfig = errors.New("configuration file should contain at least 1 service")
ErrConfigFileNotFound = errors.New("configuration file not found")
ErrConfigNotLoaded = errors.New("configuration is nil")
config *Config
ErrNoServiceInConfig = errors.New("configuration file should contain at least 1 service")
ErrConfigFileNotFound = errors.New("configuration file not found")
ErrConfigNotLoaded = errors.New("configuration is nil")
ErrInvalidSecurityConfig = errors.New("invalid security configuration")
config *Config
)
// Config is the main configuration structure
type Config struct {
Metrics bool `yaml:"metrics"`
Debug bool `yaml:"debug"`
Security *security.Config `yaml:"security"`
Alerting *alerting.Config `yaml:"alerting"`
Services []*core.Service `yaml:"services"`
}
@@ -36,7 +43,7 @@ func Get() *Config {
}
func Load(configFile string) error {
log.Printf("[config][Load] Attempting to load config from configFile=%s", configFile)
log.Printf("[config][Load] Reading configuration from configFile=%s", configFile)
cfg, err := readConfigurationFile(configFile)
if err != nil {
if os.IsNotExist(err) {
@@ -75,13 +82,94 @@ func parseAndValidateConfigBytes(yamlBytes []byte) (config *Config, err error) {
// Parse configuration file
err = yaml.Unmarshal(yamlBytes, &config)
// Check if the configuration file at least has services.
if config == nil || len(config.Services) == 0 {
if config == nil || config.Services == nil || len(config.Services) == 0 {
err = ErrNoServiceInConfig
} else {
// Set the default values if they aren't set
for _, service := range config.Services {
service.Validate()
}
validateAlertingConfig(config)
validateSecurityConfig(config)
validateServicesConfig(config)
}
return
}
func validateServicesConfig(config *Config) {
for _, service := range config.Services {
if config.Debug {
log.Printf("[config][validateServicesConfig] Validating service '%s'", service.Name)
}
service.ValidateAndSetDefaults()
}
log.Printf("[config][validateServicesConfig] Validated %d services", len(config.Services))
}
func validateSecurityConfig(config *Config) {
if config.Security != nil {
if config.Security.IsValid() {
if config.Debug {
log.Printf("[config][validateSecurityConfig] Basic security configuration has been validated")
}
} else {
// If there was an attempt to configure security, then it must mean that some confidential or private
// data are exposed. As a result, we'll force a panic because it's better to be safe than sorry.
panic(ErrInvalidSecurityConfig)
}
}
}
func validateAlertingConfig(config *Config) {
if config.Alerting == nil {
log.Printf("[config][validateAlertingConfig] Alerting is not configured")
return
}
alertTypes := []core.AlertType{
core.SlackAlert,
core.TwilioAlert,
core.PagerDutyAlert,
core.CustomAlert,
}
var validProviders, invalidProviders []core.AlertType
for _, alertType := range alertTypes {
alertProvider := GetAlertingProviderByAlertType(config, alertType)
if alertProvider != nil {
if alertProvider.IsValid() {
validProviders = append(validProviders, alertType)
} else {
log.Printf("[config][validateAlertingConfig] Ignoring provider=%s because configuration is invalid", alertType)
invalidProviders = append(invalidProviders, alertType)
}
} else {
invalidProviders = append(invalidProviders, alertType)
}
}
log.Printf("[config][validateAlertingConfig] configuredProviders=%s; ignoredProviders=%s", validProviders, invalidProviders)
}
func GetAlertingProviderByAlertType(config *Config, alertType core.AlertType) provider.AlertProvider {
switch alertType {
case core.SlackAlert:
if config.Alerting.Slack == nil {
// Since we're returning an interface, we need to explicitly return nil, even if the provider itself is nil
return nil
}
return config.Alerting.Slack
case core.TwilioAlert:
if config.Alerting.Twilio == nil {
// Since we're returning an interface, we need to explicitly return nil, even if the provider itself is nil
return nil
}
return config.Alerting.Twilio
case core.PagerDutyAlert:
if config.Alerting.PagerDuty == nil {
// Since we're returning an interface, we need to explicitly return nil, even if the provider itself is nil
return nil
}
return config.Alerting.PagerDuty
case core.CustomAlert:
if config.Alerting.Custom == nil {
// Since we're returning an interface, we need to explicitly return nil, even if the provider itself is nil
return nil
}
return config.Alerting.Custom
}
return nil
}

View File

@@ -36,7 +36,6 @@ services:
if config.Services[1].Url != "https://api.github.com/healthz" {
t.Errorf("URL should have been %s", "https://api.github.com/healthz")
}
fmt.Println(config.Services[0].Interval)
if config.Services[0].Interval != 15*time.Second {
t.Errorf("Interval should have been %s", 15*time.Second)
}
@@ -123,6 +122,8 @@ func TestParseAndValidateConfigBytesWithAlerting(t *testing.T) {
alerting:
slack:
webhook-url: "http://example.com"
pagerduty:
integration-key: "00000000000000000000000000000000"
services:
- name: twinnation
url: https://twinnation.org/actuator/health
@@ -144,10 +145,19 @@ services:
t.Error("Metrics should've been false by default")
}
if config.Alerting == nil {
t.Fatal("config.AlertingConfig shouldn't have been nil")
t.Fatal("config.Alerting shouldn't have been nil")
}
if config.Alerting.Slack == nil || !config.Alerting.Slack.IsValid() {
t.Fatal("Slack alerting config should've been valid")
}
if config.Alerting.Slack.WebhookUrl != "http://example.com" {
t.Errorf("Slack webhook should've been %s, but was %s", "http://example.com", config.Alerting.Slack)
t.Errorf("Slack webhook should've been %s, but was %s", "http://example.com", config.Alerting.Slack.WebhookUrl)
}
if config.Alerting.PagerDuty == nil || !config.Alerting.PagerDuty.IsValid() {
t.Fatal("PagerDuty alerting config should've been valid")
}
if config.Alerting.PagerDuty.IntegrationKey != "00000000000000000000000000000000" {
t.Errorf("PagerDuty integration key should've been %s, but was %s", "00000000000000000000000000000000", config.Alerting.PagerDuty.IntegrationKey)
}
if len(config.Services) != 1 {
t.Error("There should've been 1 service")
@@ -180,3 +190,84 @@ services:
t.Errorf("The type of the alert should've been %s, but it was %s", "Healthcheck failed 7 times in a row", config.Services[0].Alerts[0].Description)
}
}
func TestParseAndValidateConfigBytesWithInvalidPagerDutyAlertingConfig(t *testing.T) {
config, err := parseAndValidateConfigBytes([]byte(`
alerting:
pagerduty:
integration-key: "INVALID_KEY"
services:
- name: twinnation
url: https://twinnation.org/actuator/health
conditions:
- "[STATUS] == 200"
`))
if err != nil {
t.Error("No error should've been returned")
}
if config == nil {
t.Fatal("Config shouldn't have been nil")
}
if config.Alerting == nil {
t.Fatal("config.Alerting shouldn't have been nil")
}
if config.Alerting.PagerDuty == nil {
t.Fatal("PagerDuty alerting config shouldn't have been nil")
}
if config.Alerting.PagerDuty.IsValid() {
t.Fatal("PagerDuty alerting config should've been invalid")
}
}
func TestParseAndValidateConfigBytesWithInvalidSecurityConfig(t *testing.T) {
defer func() { recover() }()
_, _ = parseAndValidateConfigBytes([]byte(`
security:
basic:
username: "admin"
password-sha512: "invalid-sha512-hash"
services:
- name: twinnation
url: https://twinnation.org/actuator/health
conditions:
- "[STATUS] == 200"
`))
t.Error("Function should've panicked")
}
func TestParseAndValidateConfigBytesWithValidSecurityConfig(t *testing.T) {
const expectedUsername = "admin"
const expectedPasswordHash = "6b97ed68d14eb3f1aa959ce5d49c7dc612e1eb1dafd73b1e705847483fd6a6c809f2ceb4e8df6ff9984c6298ff0285cace6614bf8daa9f0070101b6c89899e22"
config, err := parseAndValidateConfigBytes([]byte(fmt.Sprintf(`
security:
basic:
username: "%s"
password-sha512: "%s"
services:
- name: twinnation
url: https://twinnation.org/actuator/health
conditions:
- "[STATUS] == 200"
`, expectedUsername, expectedPasswordHash)))
if err != nil {
t.Error("No error should've been returned")
}
if config == nil {
t.Fatal("Config shouldn't have been nil")
}
if config.Security == nil {
t.Fatal("config.Security shouldn't have been nil")
}
if !config.Security.IsValid() {
t.Error("Security config should've been valid")
}
if config.Security.Basic == nil {
t.Fatal("config.Security.Basic shouldn't have been nil")
}
if config.Security.Basic.Username != expectedUsername {
t.Errorf("config.Security.Basic.Username should've been %s, but was %s", expectedUsername, config.Security.Basic.Username)
}
if config.Security.Basic.PasswordSha512Hash != expectedPasswordHash {
t.Errorf("config.Security.Basic.PasswordSha512Hash should've been %s, but was %s", expectedPasswordHash, config.Security.Basic.PasswordSha512Hash)
}
}

View File

@@ -2,23 +2,61 @@ package core
import (
"fmt"
"github.com/TwinProduction/gatus/jsonpath"
"github.com/TwinProduction/gatus/pattern"
"log"
"strconv"
"strings"
)
const (
// StatusPlaceholder is a placeholder for a HTTP status.
//
// Values that could replace the placeholder: 200, 404, 500, ...
StatusPlaceholder = "[STATUS]"
// IPPlaceHolder is a placeholder for an IP.
//
// Values that could replace the placeholder: 127.0.0.1, 10.0.0.1, ...
IPPlaceHolder = "[IP]"
// ResponseTimePlaceHolder is a placeholder for the request response time, in milliseconds.
//
// Values that could replace the placeholder: 1, 500, 1000, ...
ResponseTimePlaceHolder = "[RESPONSE_TIME]"
// BodyPlaceHolder is a placeholder for the body of the response
//
// Values that could replace the placeholder: {}, {"data":{"name":"john"}}, ...
BodyPlaceHolder = "[BODY]"
// ConnectedPlaceHolder is a placeholder for whether a connection was successfully established.
//
// Values that could replace the placeholder: true, false
ConnectedPlaceHolder = "[CONNECTED]"
LengthFunctionPrefix = "len("
PatternFunctionPrefix = "pat("
FunctionSuffix = ")"
InvalidConditionElementSuffix = "(INVALID)"
)
// Condition is a condition that needs to be met in order for a Service to be considered healthy.
type Condition string
// evaluate the Condition with the Result of the health check
func (c *Condition) evaluate(result *Result) bool {
condition := string(*c)
success := false
var resolvedCondition string
if strings.Contains(condition, "==") {
parts := sanitizeAndResolve(strings.Split(condition, "=="), result)
success = parts[0] == parts[1]
success = isEqual(parts[0], parts[1])
resolvedCondition = fmt.Sprintf("%v == %v", parts[0], parts[1])
} else if strings.Contains(condition, "!=") {
parts := sanitizeAndResolve(strings.Split(condition, "!="), result)
success = parts[0] != parts[1]
success = !isEqual(parts[0], parts[1])
resolvedCondition = fmt.Sprintf("%v != %v", parts[0], parts[1])
} else if strings.Contains(condition, "<=") {
parts := sanitizeAndResolveNumerical(strings.Split(condition, "<="), result)
@@ -49,3 +87,83 @@ func (c *Condition) evaluate(result *Result) bool {
result.ConditionResults = append(result.ConditionResults, &ConditionResult{Condition: conditionToDisplay, Success: success})
return success
}
// isEqual compares two strings.
//
// It also supports the pattern function. That is to say, if one of the strings starts with PatternFunctionPrefix
// and ends with FunctionSuffix, it will be treated like a pattern.
func isEqual(first, second string) bool {
var isFirstPattern, isSecondPattern bool
if strings.HasPrefix(first, PatternFunctionPrefix) && strings.HasSuffix(first, FunctionSuffix) {
isFirstPattern = true
first = strings.TrimSuffix(strings.TrimPrefix(first, PatternFunctionPrefix), FunctionSuffix)
}
if strings.HasPrefix(second, PatternFunctionPrefix) && strings.HasSuffix(second, FunctionSuffix) {
isSecondPattern = true
second = strings.TrimSuffix(strings.TrimPrefix(second, PatternFunctionPrefix), FunctionSuffix)
}
if isFirstPattern && !isSecondPattern {
return pattern.Match(first, second)
} else if !isFirstPattern && isSecondPattern {
return pattern.Match(second, first)
} else {
return first == second
}
}
// sanitizeAndResolve sanitizes and resolves a list of element and returns the list of resolved elements
func sanitizeAndResolve(list []string, result *Result) []string {
var sanitizedList []string
body := strings.TrimSpace(string(result.Body))
for _, element := range list {
element = strings.TrimSpace(element)
switch strings.ToUpper(element) {
case StatusPlaceholder:
element = strconv.Itoa(result.HttpStatus)
case IPPlaceHolder:
element = result.Ip
case ResponseTimePlaceHolder:
element = strconv.Itoa(int(result.Duration.Milliseconds()))
case BodyPlaceHolder:
element = body
case ConnectedPlaceHolder:
element = strconv.FormatBool(result.Connected)
default:
// if contains the BodyPlaceHolder, then evaluate json path
if strings.Contains(element, BodyPlaceHolder) {
wantLength := false
if strings.HasPrefix(element, LengthFunctionPrefix) && strings.HasSuffix(element, FunctionSuffix) {
wantLength = true
element = strings.TrimSuffix(strings.TrimPrefix(element, LengthFunctionPrefix), FunctionSuffix)
}
resolvedElement, resolvedElementLength, err := jsonpath.Eval(strings.Replace(element, fmt.Sprintf("%s.", BodyPlaceHolder), "", 1), result.Body)
if err != nil {
result.Errors = append(result.Errors, err.Error())
element = fmt.Sprintf("%s %s", element, InvalidConditionElementSuffix)
} else {
if wantLength {
element = fmt.Sprintf("%d", resolvedElementLength)
} else {
element = resolvedElement
}
}
}
}
sanitizedList = append(sanitizedList, element)
}
return sanitizedList
}
func sanitizeAndResolveNumerical(list []string, result *Result) []int {
var sanitizedNumbers []int
sanitizedList := sanitizeAndResolve(list, result)
for _, element := range sanitizedList {
if number, err := strconv.Atoi(element); err != nil {
// Default to 0 if the string couldn't be converted to an integer
sanitizedNumbers = append(sanitizedNumbers, 0)
} else {
sanitizedNumbers = append(sanitizedNumbers, number)
}
}
return sanitizedNumbers
}

View File

@@ -113,6 +113,24 @@ func TestCondition_evaluateWithBodyJsonPathComplex(t *testing.T) {
}
}
func TestCondition_evaluateWithBodyJsonPathDoublePlaceholders(t *testing.T) {
condition := Condition("[BODY].user.firstName != [BODY].user.lastName")
result := &Result{Body: []byte("{\"user\": {\"firstName\": \"john\", \"lastName\": \"doe\"}}")}
condition.evaluate(result)
if !result.ConditionResults[0].Success {
t.Errorf("Condition '%s' should have been a success", condition)
}
}
func TestCondition_evaluateWithBodyJsonPathDoublePlaceholdersFailure(t *testing.T) {
condition := Condition("[BODY].user.firstName == [BODY].user.lastName")
result := &Result{Body: []byte("{\"user\": {\"firstName\": \"john\", \"lastName\": \"doe\"}}")}
condition.evaluate(result)
if result.ConditionResults[0].Success {
t.Errorf("Condition '%s' should have been a failure", condition)
}
}
func TestCondition_evaluateWithBodyJsonPathLongInt(t *testing.T) {
condition := Condition("[BODY].data.id == 1")
result := &Result{Body: []byte("{\"data\": {\"id\": 1}}")}
@@ -184,3 +202,84 @@ func TestCondition_evaluateWithBodyStringLength(t *testing.T) {
t.Errorf("Condition '%s' should have been a success", condition)
}
}
func TestCondition_evaluateWithBodyStringPattern(t *testing.T) {
condition := Condition("[BODY].name == pat(*ohn*)")
result := &Result{Body: []byte("{\"name\": \"john.doe\"}")}
condition.evaluate(result)
if !result.ConditionResults[0].Success {
t.Errorf("Condition '%s' should have been a success", condition)
}
}
func TestCondition_evaluateWithBodyStringPatternFailure(t *testing.T) {
condition := Condition("[BODY].name == pat(bob*)")
result := &Result{Body: []byte("{\"name\": \"john.doe\"}")}
condition.evaluate(result)
if result.ConditionResults[0].Success {
t.Errorf("Condition '%s' should have been a failure", condition)
}
}
func TestCondition_evaluateWithBodyPatternFailure(t *testing.T) {
condition := Condition("[BODY] == pat(*john*)")
result := &Result{Body: []byte("{\"name\": \"john.doe\"}")}
condition.evaluate(result)
if !result.ConditionResults[0].Success {
t.Errorf("Condition '%s' should have been a success", condition)
}
}
func TestCondition_evaluateWithIPPattern(t *testing.T) {
condition := Condition("[IP] == pat(10.*)")
result := &Result{Ip: "10.0.0.0"}
condition.evaluate(result)
if !result.ConditionResults[0].Success {
t.Errorf("Condition '%s' should have been a success", condition)
}
}
func TestCondition_evaluateWithIPPatternFailure(t *testing.T) {
condition := Condition("[IP] == pat(10.*)")
result := &Result{Ip: "255.255.255.255"}
condition.evaluate(result)
if result.ConditionResults[0].Success {
t.Errorf("Condition '%s' should have been a failure", condition)
}
}
func TestCondition_evaluateWithStatusPattern(t *testing.T) {
condition := Condition("[STATUS] == pat(4*)")
result := &Result{HttpStatus: 404}
condition.evaluate(result)
if !result.ConditionResults[0].Success {
t.Errorf("Condition '%s' should have been a success", condition)
}
}
func TestCondition_evaluateWithStatusPatternFailure(t *testing.T) {
condition := Condition("[STATUS] != pat(4*)")
result := &Result{HttpStatus: 404}
condition.evaluate(result)
if result.ConditionResults[0].Success {
t.Errorf("Condition '%s' should have been a failure", condition)
}
}
func TestCondition_evaluateWithConnected(t *testing.T) {
condition := Condition("[CONNECTED] == true")
result := &Result{Connected: true}
condition.evaluate(result)
if !result.ConditionResults[0].Success {
t.Errorf("Condition '%s' should have been a success", condition)
}
}
func TestCondition_evaluateWithConnectedFailure(t *testing.T) {
condition := Condition("[CONNECTED] == true")
result := &Result{Connected: false}
condition.evaluate(result)
if result.ConditionResults[0].Success {
t.Errorf("Condition '%s' should have been a failure", condition)
}
}

View File

@@ -9,6 +9,7 @@ import (
"net"
"net/http"
"net/url"
"strings"
"time"
)
@@ -46,11 +47,15 @@ type Service struct {
// Alerts is the alerting configuration for the service in case of failure
Alerts []*Alert `yaml:"alerts"`
// Insecure is whether to skip verifying the server's certificate chain and host name
Insecure bool `yaml:"insecure,omitempty"`
NumberOfFailuresInARow int
NumberOfSuccessesInARow int
}
func (service *Service) Validate() {
// ValidateAndSetDefaults validates the service's configuration and sets the default value of fields that have one
func (service *Service) ValidateAndSetDefaults() {
// Set default values
if service.Interval == 0 {
service.Interval = 1 * time.Minute
@@ -83,7 +88,8 @@ func (service *Service) Validate() {
}
}
func (service *Service) EvaluateConditions() *Result {
// EvaluateHealth sends a request to the service's URL and evaluates the conditions of the service.
func (service *Service) EvaluateHealth() *Result {
result := &Result{Success: true, Errors: []string{}}
service.getIp(result)
if len(result.Errors) == 0 {
@@ -131,19 +137,30 @@ func (service *Service) getIp(result *Result) {
}
func (service *Service) call(result *Result) {
request := service.buildRequest()
startTime := time.Now()
response, err := client.GetHttpClient().Do(request)
if err != nil {
result.Duration = time.Since(startTime)
result.Errors = append(result.Errors, err.Error())
return
isServiceTcp := strings.HasPrefix(service.Url, "tcp://")
var request *http.Request
var response *http.Response
var err error
if !isServiceTcp {
request = service.buildRequest()
}
result.Duration = time.Since(startTime)
result.HttpStatus = response.StatusCode
result.Body, err = ioutil.ReadAll(response.Body)
if err != nil {
result.Errors = append(result.Errors, err.Error())
startTime := time.Now()
if isServiceTcp {
result.Connected = client.CanCreateConnectionToTcpService(strings.TrimPrefix(service.Url, "tcp://"))
result.Duration = time.Since(startTime)
} else {
response, err = client.GetHttpClient(service.Insecure).Do(request)
result.Duration = time.Since(startTime)
if err != nil {
result.Errors = append(result.Errors, err.Error())
return
}
result.HttpStatus = response.StatusCode
result.Connected = response.StatusCode > 0
result.Body, err = ioutil.ReadAll(response.Body)
if err != nil {
result.Errors = append(result.Errors, err.Error())
}
}
}

View File

@@ -4,33 +4,39 @@ import (
"testing"
)
func TestIntegrationEvaluateConditions(t *testing.T) {
func TestIntegrationEvaluateHealth(t *testing.T) {
condition := Condition("[STATUS] == 200")
service := Service{
Name: "TwiNNatioN",
Url: "https://twinnation.org/health",
Conditions: []*Condition{&condition},
}
result := service.EvaluateConditions()
result := service.EvaluateHealth()
if !result.ConditionResults[0].Success {
t.Errorf("Condition '%s' should have been a success", condition)
}
if !result.Connected {
t.Error("Because the connection has been established, result.Connected should've been true")
}
if !result.Success {
t.Error("Because all conditions passed, this should have been a success")
}
}
func TestIntegrationEvaluateConditionsWithFailure(t *testing.T) {
func TestIntegrationEvaluateHealthWithFailure(t *testing.T) {
condition := Condition("[STATUS] == 500")
service := Service{
Name: "TwiNNatioN",
Url: "https://twinnation.org/health",
Conditions: []*Condition{&condition},
}
result := service.EvaluateConditions()
result := service.EvaluateHealth()
if result.ConditionResults[0].Success {
t.Errorf("Condition '%s' should have been a failure", condition)
}
if !result.Connected {
t.Error("Because the connection has been established, result.Connected should've been true")
}
if result.Success {
t.Error("Because one of the conditions failed, success should have been false")
}

View File

@@ -9,16 +9,37 @@ type HealthStatus struct {
Message string `json:"message,omitempty"`
}
// Result of the evaluation of a Service
type Result struct {
HttpStatus int `json:"status"`
Body []byte `json:"-"`
Hostname string `json:"hostname"`
Ip string `json:"-"`
Duration time.Duration `json:"duration"`
Errors []string `json:"errors"`
// HttpStatus is the HTTP response status code
HttpStatus int `json:"status"`
// Body is the response body
Body []byte `json:"-"`
// Hostname extracted from the Service Url
Hostname string `json:"hostname"`
// Ip resolved from the Service Url
Ip string `json:"-"`
// Connected whether a connection to the host was established successfully
Connected bool `json:"-"`
// Duration time that the request took
Duration time.Duration `json:"duration"`
// Errors encountered during the evaluation of the service's health
Errors []string `json:"errors"`
// ConditionResults results of the service's conditions
ConditionResults []*ConditionResult `json:"condition-results"`
Success bool `json:"success"`
Timestamp time.Time `json:"timestamp"`
// Success whether the result signifies a success or not
Success bool `json:"success"`
// Timestamp when the request was sent
Timestamp time.Time `json:"timestamp"`
}
type ConditionResult struct {

View File

@@ -1,74 +0,0 @@
package core
import (
"fmt"
"github.com/TwinProduction/gatus/jsonpath"
"strconv"
"strings"
)
const (
StatusPlaceholder = "[STATUS]"
IPPlaceHolder = "[IP]"
ResponseTimePlaceHolder = "[RESPONSE_TIME]"
BodyPlaceHolder = "[BODY]"
LengthFunctionPrefix = "len("
FunctionSuffix = ")"
InvalidConditionElementSuffix = "(INVALID)"
)
func sanitizeAndResolve(list []string, result *Result) []string {
var sanitizedList []string
body := strings.TrimSpace(string(result.Body))
for _, element := range list {
element = strings.TrimSpace(element)
switch strings.ToUpper(element) {
case StatusPlaceholder:
element = strconv.Itoa(result.HttpStatus)
case IPPlaceHolder:
element = result.Ip
case ResponseTimePlaceHolder:
element = strconv.Itoa(int(result.Duration.Milliseconds()))
case BodyPlaceHolder:
element = body
default:
// if starts with BodyPlaceHolder, then evaluate json path
if strings.Contains(element, BodyPlaceHolder) {
wantLength := false
if strings.HasPrefix(element, LengthFunctionPrefix) && strings.HasSuffix(element, FunctionSuffix) {
wantLength = true
element = strings.TrimSuffix(strings.TrimPrefix(element, LengthFunctionPrefix), FunctionSuffix)
}
resolvedElement, resolvedElementLength, err := jsonpath.Eval(strings.Replace(element, fmt.Sprintf("%s.", BodyPlaceHolder), "", 1), result.Body)
if err != nil {
result.Errors = append(result.Errors, err.Error())
element = fmt.Sprintf("%s %s", element, InvalidConditionElementSuffix)
} else {
if wantLength {
element = fmt.Sprintf("%d", resolvedElementLength)
} else {
element = resolvedElement
}
}
}
}
sanitizedList = append(sanitizedList, element)
}
return sanitizedList
}
func sanitizeAndResolveNumerical(list []string, result *Result) []int {
var sanitizedNumbers []int
sanitizedList := sanitizeAndResolve(list, result)
for _, element := range sanitizedList {
if number, err := strconv.Atoi(element); err != nil {
// Default to 0 if the string couldn't be converted to an integer
sanitizedNumbers = append(sanitizedNumbers, 0)
} else {
sanitizedNumbers = append(sanitizedNumbers, number)
}
}
return sanitizedNumbers
}

View File

@@ -13,6 +13,16 @@ data:
interval: 5m
conditions:
- "[STATUS] == 200"
- name: cat-fact
url: "https://cat-fact.herokuapp.com/facts/random"
interval: 5m
conditions:
- "[STATUS] == 200"
- "[BODY].deleted == false"
- "len([BODY].text) > 0"
- "[BODY].text == pat(*cat*)"
- "[STATUS] == pat(2*)"
- "[CONNECTED] == true"
- name: Example
url: https://example.com/
conditions:
@@ -49,10 +59,10 @@ spec:
protocol: TCP
resources:
limits:
cpu: 50m
cpu: 200m
memory: 50M
requests:
cpu: 20m
cpu: 50m
memory: 20M
volumeMounts:
- mountPath: /config

View File

@@ -4,6 +4,7 @@ import (
"bytes"
"compress/gzip"
"github.com/TwinProduction/gatus/config"
"github.com/TwinProduction/gatus/security"
"github.com/TwinProduction/gatus/watchdog"
"github.com/prometheus/client_golang/prometheus/promhttp"
"log"
@@ -23,7 +24,11 @@ var (
func main() {
cfg := loadConfiguration()
http.HandleFunc("/api/v1/results", serviceResultsHandler)
resultsHandler := serviceResultsHandler
if cfg.Security != nil && cfg.Security.IsValid() {
resultsHandler = security.Handler(serviceResultsHandler, cfg.Security)
}
http.HandleFunc("/api/v1/results", resultsHandler)
http.HandleFunc("/health", healthHandler)
http.Handle("/", GzipHandler(http.FileServer(http.Dir("./static"))))
if cfg.Metrics {

12
pattern/pattern.go Normal file
View File

@@ -0,0 +1,12 @@
package pattern
import "path/filepath"
// Match checks whether a string matches a pattern
func Match(pattern, s string) bool {
if pattern == "*" {
return true
}
matched, _ := filepath.Match(pattern, s)
return matched
}

37
pattern/pattern_test.go Normal file
View File

@@ -0,0 +1,37 @@
package pattern
import "testing"
func TestMatch(t *testing.T) {
testMatch(t, "*", "livingroom_123", true)
testMatch(t, "**", "livingroom_123", true)
testMatch(t, "living*", "livingroom_123", true)
testMatch(t, "*living*", "livingroom_123", true)
testMatch(t, "*123", "livingroom_123", true)
testMatch(t, "*_*", "livingroom_123", true)
testMatch(t, "living*_*3", "livingroom_123", true)
testMatch(t, "living*room_*3", "livingroom_123", true)
testMatch(t, "living*room_*3", "livingroom_123", true)
testMatch(t, "*vin*om*2*", "livingroom_123", true)
testMatch(t, "livingroom_123", "livingroom_123", true)
testMatch(t, "*livingroom_123*", "livingroom_123", true)
testMatch(t, "livingroom", "livingroom_123", false)
testMatch(t, "livingroom123", "livingroom_123", false)
testMatch(t, "what", "livingroom_123", false)
testMatch(t, "*what*", "livingroom_123", false)
testMatch(t, "*.*", "livingroom_123", false)
testMatch(t, "room*123", "livingroom_123", false)
}
func testMatch(t *testing.T, pattern, key string, expectedToMatch bool) {
matched := Match(pattern, key)
if expectedToMatch {
if !matched {
t.Errorf("%s should've matched pattern '%s'", key, pattern)
}
} else {
if matched {
t.Errorf("%s shouldn't have matched pattern '%s'", key, pattern)
}
}
}

19
security/handler.go Normal file
View File

@@ -0,0 +1,19 @@
package security
import (
"net/http"
"strings"
)
func Handler(handler http.HandlerFunc, security *Config) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
usernameEntered, passwordEntered, ok := r.BasicAuth()
if !ok || usernameEntered != security.Basic.Username || Sha512(passwordEntered) != strings.ToLower(security.Basic.PasswordSha512Hash) {
w.Header().Set("WWW-Authenticate", "Basic")
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte("Unauthorized"))
return
}
handler(w, r)
}
}

58
security/handler_test.go Normal file
View File

@@ -0,0 +1,58 @@
package security
import (
"net/http"
"net/http/httptest"
"testing"
)
func mockHandler(writer http.ResponseWriter, _ *http.Request) {
writer.WriteHeader(200)
}
func TestHandlerWhenNotAuthenticated(t *testing.T) {
handler := Handler(mockHandler, &Config{&BasicConfig{
Username: "john.doe",
PasswordSha512Hash: "6b97ed68d14eb3f1aa959ce5d49c7dc612e1eb1dafd73b1e705847483fd6a6c809f2ceb4e8df6ff9984c6298ff0285cace6614bf8daa9f0070101b6c89899e22",
}})
request, _ := http.NewRequest("GET", "/api/v1/results", nil)
responseRecorder := httptest.NewRecorder()
handler.ServeHTTP(responseRecorder, request)
if responseRecorder.Code != http.StatusUnauthorized {
t.Error("Expected code to be 401, but was", responseRecorder.Code)
}
}
func TestHandlerWhenAuthenticated(t *testing.T) {
handler := Handler(mockHandler, &Config{&BasicConfig{
Username: "john.doe",
PasswordSha512Hash: "6b97ed68d14eb3f1aa959ce5d49c7dc612e1eb1dafd73b1e705847483fd6a6c809f2ceb4e8df6ff9984c6298ff0285cace6614bf8daa9f0070101b6c89899e22",
}})
request, _ := http.NewRequest("GET", "/api/v1/results", nil)
request.SetBasicAuth("john.doe", "hunter2")
responseRecorder := httptest.NewRecorder()
handler.ServeHTTP(responseRecorder, request)
if responseRecorder.Code != http.StatusOK {
t.Error("Expected code to be 200, but was", responseRecorder.Code)
}
}
func TestHandlerWhenAuthenticatedWithBadCredentials(t *testing.T) {
handler := Handler(mockHandler, &Config{&BasicConfig{
Username: "john.doe",
PasswordSha512Hash: "6b97ed68d14eb3f1aa959ce5d49c7dc612e1eb1dafd73b1e705847483fd6a6c809f2ceb4e8df6ff9984c6298ff0285cace6614bf8daa9f0070101b6c89899e22",
}})
request, _ := http.NewRequest("GET", "/api/v1/results", nil)
request.SetBasicAuth("john.doe", "bad-password")
responseRecorder := httptest.NewRecorder()
handler.ServeHTTP(responseRecorder, request)
if responseRecorder.Code != http.StatusUnauthorized {
t.Error("Expected code to be 401, but was", responseRecorder.Code)
}
}

18
security/security.go Normal file
View File

@@ -0,0 +1,18 @@
package security
type Config struct {
Basic *BasicConfig `yaml:"basic"`
}
func (c *Config) IsValid() bool {
return c.Basic != nil && c.Basic.IsValid()
}
type BasicConfig struct {
Username string `yaml:"username"`
PasswordSha512Hash string `yaml:"password-sha512"`
}
func (c *BasicConfig) IsValid() bool {
return len(c.Username) > 0 && len(c.PasswordSha512Hash) == 128
}

23
security/security_test.go Normal file
View File

@@ -0,0 +1,23 @@
package security
import "testing"
func TestBasicConfig_IsValid(t *testing.T) {
basicConfig := &BasicConfig{
Username: "admin",
PasswordSha512Hash: Sha512("test"),
}
if !basicConfig.IsValid() {
t.Error("basicConfig should've been valid")
}
}
func TestBasicConfig_IsValidWhenPasswordIsInvalid(t *testing.T) {
basicConfig := &BasicConfig{
Username: "admin",
PasswordSha512Hash: "",
}
if basicConfig.IsValid() {
t.Error("basicConfig shouldn't have been valid")
}
}

12
security/sha512.go Normal file
View File

@@ -0,0 +1,12 @@
package security
import (
"crypto/sha512"
"fmt"
)
func Sha512(s string) string {
hash := sha512.New()
hash.Write([]byte(s))
return fmt.Sprintf("%x", hash.Sum(nil))
}

12
security/sha512_test.go Normal file
View File

@@ -0,0 +1,12 @@
package security
import "testing"
func TestSha512(t *testing.T) {
input := "password"
expectedHash := "b109f3bbbc244eb82441917ed06d618b9008dd09b3befd1b5e07394c706a8bb980b1d7785e5976ec049b46df5f1326af5a2ea6d103fd07c95385ffab0cacbc86"
hash := Sha512(input)
if hash != expectedHash {
t.Errorf("Expected hash to be '%s', but was '%s'", expectedHash, hash)
}
}

View File

@@ -90,6 +90,15 @@
#social img:hover {
opacity: 1;
}
#settings {
position: fixed;
left: 5px;
bottom: 5px;
padding: 5px;
}
#settings select:focus {
box-shadow: none;
}
</style>
</head>
<body>
@@ -128,9 +137,26 @@
</a>
</div>
<div id="settings">
<div class="input-group input-group-sm">
<div class="input-group-prepend">
<div class="input-group-text">&#x21bb;</div>
</div>
<select class="form-control form-control-sm" id="refresh-rate">
<option value="10">10s</option>
<option value="30" selected>30s</option>
<option value="60">1m</option>
<option value="120">2m</option>
<option value="300">5m</option>
<option value="600">10m</option>
</select>
</div>
</div>
<script>
let serviceStatuses = {};
let timerHandler = 0;
let refreshIntervalHandler = 0;
let userClickedStatus = false;
function showTooltip(serviceName, index, element) {
@@ -291,10 +317,19 @@
.replace(/'/g, '&apos;');
}
refreshResults();
setInterval(function() {
function setRefreshInterval(seconds) {
refreshResults();
}, 30000);
refreshIntervalHandler = setInterval(function() {
refreshResults();
}, seconds * 1000)
}
$("#refresh-rate").change(function() {
clearInterval(refreshIntervalHandler);
setRefreshInterval($(this).val())
});
setRefreshInterval(30);
$("#refresh-rate").val(30);
</script>
</body>
</html>

View File

@@ -2,8 +2,6 @@ package watchdog
import (
"encoding/json"
"fmt"
"github.com/TwinProduction/gatus/alerting/provider/custom"
"github.com/TwinProduction/gatus/config"
"github.com/TwinProduction/gatus/core"
"log"
@@ -32,46 +30,20 @@ func handleAlertsToTrigger(service *core.Service, result *core.Result, cfg *conf
}
if alert.Triggered {
if cfg.Debug {
log.Printf("[watchdog][handleAlertsToTrigger] Alert with description='%s' has already been triggered, skipping", alert.Description)
log.Printf("[watchdog][handleAlertsToTrigger] Alert with description='%s' has already been TRIGGERED, skipping", alert.Description)
}
continue
}
var alertProvider *custom.AlertProvider
if alert.Type == core.SlackAlert {
if cfg.Alerting.Slack != nil && cfg.Alerting.Slack.IsValid() {
log.Printf("[watchdog][handleAlertsToTrigger] Sending Slack alert because alert with description='%s' has been triggered", alert.Description)
alertProvider = cfg.Alerting.Slack.ToCustomAlertProvider(service, alert, result, false)
} else {
log.Printf("[watchdog][handleAlertsToTrigger] Not sending Slack alert despite being triggered, because there is no Slack webhook configured")
}
} else if alert.Type == core.PagerDutyAlert {
if cfg.Alerting.PagerDuty != nil && cfg.Alerting.PagerDuty.IsValid() {
log.Printf("[watchdog][handleAlertsToTrigger] Sending PagerDuty alert because alert with description='%s' has been triggered", alert.Description)
alertProvider = cfg.Alerting.PagerDuty.ToCustomAlertProvider("trigger", "", service, fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description))
} else {
log.Printf("[watchdog][handleAlertsToTrigger] Not sending PagerDuty alert despite being triggered, because PagerDuty isn't configured properly")
}
} else if alert.Type == core.TwilioAlert {
if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() {
log.Printf("[watchdog][handleAlertsToTrigger] Sending Twilio alert because alert with description='%s' has been triggered", alert.Description)
alertProvider = cfg.Alerting.Twilio.ToCustomAlertProvider(fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description))
} else {
log.Printf("[watchdog][handleAlertsToTrigger] Not sending Twilio alert despite being triggered, because Twilio config settings missing")
}
} else if alert.Type == core.CustomAlert {
if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() {
log.Printf("[watchdog][handleAlertsToTrigger] Sending custom alert because alert with description='%s' has been triggered", alert.Description)
alertProvider = cfg.Alerting.Custom
} else {
log.Printf("[watchdog][handleAlertsToTrigger] Not sending custom alert despite being triggered, because there is no custom url configured")
}
}
if alertProvider != nil {
alertProvider := config.GetAlertingProviderByAlertType(cfg, alert.Type)
if alertProvider != nil && alertProvider.IsValid() {
log.Printf("[watchdog][handleAlertsToTrigger] Sending %s alert because alert with description='%s' has been TRIGGERED", alert.Type, alert.Description)
customAlertProvider := alertProvider.ToCustomAlertProvider(service, alert, result, false)
// TODO: retry on error
var err error
// We need to extract the DedupKey from PagerDuty's response
if alert.Type == core.PagerDutyAlert {
var body []byte
body, err = alertProvider.Send(service.Name, alert.Description, true)
body, err = customAlertProvider.Send(service.Name, alert.Description, false)
if err == nil {
var response pagerDutyResponse
err = json.Unmarshal(body, &response)
@@ -82,13 +54,17 @@ func handleAlertsToTrigger(service *core.Service, result *core.Result, cfg *conf
}
}
} else {
_, err = alertProvider.Send(service.Name, alert.Description, false)
// All other alert types don't need to extract anything from the body, so we can just send the request right away
_, err = customAlertProvider.Send(service.Name, alert.Description, false)
}
if err != nil {
log.Printf("[watchdog][handleAlertsToTrigger] Ran into error sending an alert: %s", err.Error())
} else {
alert.Triggered = true
}
} else {
log.Printf("[watchdog][handleAlertsToResolve] Not sending alert of type=%s despite being TRIGGERED, because the provider wasn't configured properly", alert.Type)
}
}
}
@@ -103,44 +79,12 @@ func handleAlertsToResolve(service *core.Service, result *core.Result, cfg *conf
if !alert.SendOnResolved {
continue
}
var alertProvider *custom.AlertProvider
if alert.Type == core.SlackAlert {
if cfg.Alerting.Slack != nil && cfg.Alerting.Slack.IsValid() {
log.Printf("[watchdog][handleAlertsToResolve] Sending Slack alert because alert with description='%s' has been resolved", alert.Description)
alertProvider = cfg.Alerting.Slack.ToCustomAlertProvider(service, alert, result, true)
} else {
log.Printf("[watchdog][handleAlertsToResolve] Not sending Slack alert despite being resolved, because there is no Slack webhook configured")
}
} else if alert.Type == core.PagerDutyAlert {
if cfg.Alerting.PagerDuty != nil && cfg.Alerting.PagerDuty.IsValid() {
log.Printf("[watchdog][handleAlertsToResolve] Sending PagerDuty alert because alert with description='%s' has been resolved", alert.Description)
alertProvider = cfg.Alerting.PagerDuty.ToCustomAlertProvider("resolve", alert.ResolveKey, service, fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description))
} else {
log.Printf("[watchdog][handleAlertsToResolve] Not sending PagerDuty alert despite being resolved, because PagerDuty isn't configured properly")
}
} else if alert.Type == core.TwilioAlert {
if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() {
log.Printf("[watchdog][handleAlertsToResolve] Sending Twilio alert because alert with description='%s' has been resolved", alert.Description)
alertProvider = cfg.Alerting.Twilio.ToCustomAlertProvider(fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description))
} else {
log.Printf("[watchdog][handleAlertsToResolve] Not sending Twilio alert despite being resolved, because Twilio isn't configured properly")
}
} else if alert.Type == core.CustomAlert {
if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() {
log.Printf("[watchdog][handleAlertsToResolve] Sending custom alert because alert with description='%s' has been resolved", alert.Description)
alertProvider = &custom.AlertProvider{
Url: cfg.Alerting.Custom.Url,
Method: cfg.Alerting.Custom.Method,
Body: cfg.Alerting.Custom.Body,
Headers: cfg.Alerting.Custom.Headers,
}
} else {
log.Printf("[watchdog][handleAlertsToResolve] Not sending custom alert despite being resolved, because the custom provider isn't configured properly")
}
}
if alertProvider != nil {
alertProvider := config.GetAlertingProviderByAlertType(cfg, alert.Type)
if alertProvider != nil && alertProvider.IsValid() {
log.Printf("[watchdog][handleAlertsToResolve] Sending %s alert because alert with description='%s' has been RESOLVED", alert.Type, alert.Description)
customAlertProvider := alertProvider.ToCustomAlertProvider(service, alert, result, true)
// TODO: retry on error
_, err := alertProvider.Send(service.Name, alert.Description, true)
_, err := customAlertProvider.Send(service.Name, alert.Description, true)
if err != nil {
log.Printf("[watchdog][handleAlertsToResolve] Ran into error sending an alert: %s", err.Error())
} else {
@@ -148,6 +92,8 @@ func handleAlertsToResolve(service *core.Service, result *core.Result, cfg *conf
alert.ResolveKey = ""
}
}
} else {
log.Printf("[watchdog][handleAlertsToResolve] Not sending alert of type=%s despite being RESOLVED, because the provider wasn't configured properly", alert.Type)
}
}
service.NumberOfFailuresInARow = 0

View File

@@ -50,7 +50,7 @@ func monitor(service *core.Service) {
if cfg.Debug {
log.Printf("[watchdog][monitor] Monitoring serviceName=%s", service.Name)
}
result := service.EvaluateConditions()
result := service.EvaluateHealth()
metric.PublishMetricsForService(service, result)
serviceResultsMutex.Lock()
serviceResults[service.Name] = append(serviceResults[service.Name], result)