Add better crawler metrics

internal/server.go

@@ -190,17 +190,22 @@ func (s *Server) setupMetrics() {
 			return float64(s.db.LenTokens())
 		},
 	)
-
-	// feed cache sources
-	metrics.NewGauge(
-		"cache", "sources",
-		"Number of feed sources being fetched by the global feed cache",
+	metrics.NewGaugeFunc(
+		"db", "urls",
+		"Number of database /urls keys",
+		func() float64 {
+			return float64(s.db.URLCount())
+		},
 	)
 
-	// feed cache size
-	metrics.NewGauge(
-		"cache", "feeds",
-		"Number of unique feeds in the global feed cache",
+	// Crawler stats
+	metrics.NewCounter(
+		"crawler", "crawled",
+		"Number of links crawled by the crawler",
+	)
+	metrics.NewCounter(
+		"crawler", "scraped",
+		"Number of links scraped by the crawler",
 	)
 
 	// server info
@@ -216,17 +221,6 @@ func (s *Server) setupMetrics() {
 			"commit":       spyda.Commit,
 		}).Set(1)
 
-	// old avatars
-	metrics.NewCounter(
-		"media", "old_avatar",
-		"Count of old Avtar (PNG) conversions",
-	)
-	// old media
-	metrics.NewCounter(
-		"media", "old_media",
-		"Count of old Media (PNG) served",
-	)
-
 	s.AddRoute("GET", "/metrics", metrics.Handler())
 }