Add better crawler metrics

2021-02-02 12:07:53 +10:00
parent 865e5b5750
commit 4970b16d61
2 changed files with 18 additions and 20 deletions
--- a/internal/crawler.go
+++ b/internal/crawler.go
@@ -48,6 +48,8 @@ func (c *crawler) loop() {

 			log.Debugf("found %s", link)

+			metrics.Counter("crawler", "crawled").Inc()
+
 			url := NewURL(link)
 			url.CrawledAt = time.Now()

@@ -60,6 +62,8 @@ func (c *crawler) loop() {
 				} else {
 					if err := c.db.SetURL(hash, url); err != nil {
 						log.WithError(err).Error("error recording url %s", link)
+					} else {
+						metrics.Counter("crawler", "scraped").Inc()
 					}
 				}
 			}