Add better crawler metrics

internal/crawler.go

@@ -48,6 +48,8 @@ func (c *crawler) loop() {
 
 			log.Debugf("found %s", link)
 
+			metrics.Counter("crawler", "crawled").Inc()
+
 			url := NewURL(link)
 			url.CrawledAt = time.Now()
 
@@ -60,6 +62,8 @@ func (c *crawler) loop() {
 				} else {
 					if err := c.db.SetURL(hash, url); err != nil {
 						log.WithError(err).Error("error recording url %s", link)
+					} else {
+						metrics.Counter("crawler", "scraped").Inc()
 					}
 				}
 			}

internal/server.go

@@ -190,17 +190,22 @@ func (s *Server) setupMetrics() {
 			return float64(s.db.LenTokens())
 		},
 	)
-
+	metrics.NewGaugeFunc(
-	// feed cache sources
+		"db", "urls",
-	metrics.NewGauge(
+		"Number of database /urls keys",
-		"cache", "sources",
+		func() float64 {
-		"Number of feed sources being fetched by the global feed cache",
+			return float64(s.db.URLCount())
+		},
 	)
 
-	// feed cache size
+	// Crawler stats
-	metrics.NewGauge(
+	metrics.NewCounter(
-		"cache", "feeds",
+		"crawler", "crawled",
-		"Number of unique feeds in the global feed cache",
+		"Number of links crawled by the crawler",
+	)
+	metrics.NewCounter(
+		"crawler", "scraped",
+		"Number of links scraped by the crawler",
 	)
 
 	// server info
@@ -216,17 +221,6 @@ func (s *Server) setupMetrics() {
 			"commit":       spyda.Commit,
 		}).Set(1)
 
-	// old avatars
-	metrics.NewCounter(
-		"media", "old_avatar",
-		"Count of old Avtar (PNG) conversions",
-	)
-	// old media
-	metrics.NewCounter(
-		"media", "old_media",
-		"Count of old Media (PNG) served",
-	)
-
 	s.AddRoute("GET", "/metrics", metrics.Handler())
 }