From 4970b16d61e4377d483612b9a5849e82f438ac54 Mon Sep 17 00:00:00 2001
From: James Mills <prologic@shortcircuit.net.au>
Date: Tue, 2 Feb 2021 12:07:53 +1000
Subject: [PATCH] Add better crawler metrics

---
 internal/crawler.go |  4 ++++
 internal/server.go  | 34 ++++++++++++++--------------------
 2 files changed, 18 insertions(+), 20 deletions(-)

diff --git a/internal/crawler.go b/internal/crawler.go
index 905ae28..ce8fc2c 100644
--- a/internal/crawler.go
+++ b/internal/crawler.go
@@ -48,6 +48,8 @@ func (c *crawler) loop() {
 
 			log.Debugf("found %s", link)
 
+			metrics.Counter("crawler", "crawled").Inc()
+
 			url := NewURL(link)
 			url.CrawledAt = time.Now()
 
@@ -60,6 +62,8 @@ func (c *crawler) loop() {
 				} else {
 					if err := c.db.SetURL(hash, url); err != nil {
 						log.WithError(err).Error("error recording url %s", link)
+					} else {
+						metrics.Counter("crawler", "scraped").Inc()
 					}
 				}
 			}
diff --git a/internal/server.go b/internal/server.go
index 0390bbd..894e0ae 100644
--- a/internal/server.go
+++ b/internal/server.go
@@ -190,17 +190,22 @@ func (s *Server) setupMetrics() {
 			return float64(s.db.LenTokens())
 		},
 	)
-
-	// feed cache sources
-	metrics.NewGauge(
-		"cache", "sources",
-		"Number of feed sources being fetched by the global feed cache",
+	metrics.NewGaugeFunc(
+		"db", "urls",
+		"Number of database /urls keys",
+		func() float64 {
+			return float64(s.db.URLCount())
+		},
 	)
 
-	// feed cache size
-	metrics.NewGauge(
-		"cache", "feeds",
-		"Number of unique feeds in the global feed cache",
+	// Crawler stats
+	metrics.NewCounter(
+		"crawler", "crawled",
+		"Number of links crawled by the crawler",
+	)
+	metrics.NewCounter(
+		"crawler", "scraped",
+		"Number of links scraped by the crawler",
 	)
 
 	// server info
@@ -216,17 +221,6 @@ func (s *Server) setupMetrics() {
 			"commit":       spyda.Commit,
 		}).Set(1)
 
-	// old avatars
-	metrics.NewCounter(
-		"media", "old_avatar",
-		"Count of old Avtar (PNG) conversions",
-	)
-	// old media
-	metrics.NewCounter(
-		"media", "old_media",
-		"Count of old Media (PNG) served",
-	)
-
 	s.AddRoute("GET", "/metrics", metrics.Handler())
 }