Add crawler skeleton and add handler

internal/server.go

@@ -41,6 +41,9 @@ type Server struct {
 	router  *Router
 	server  *http.Server
 
+	// Crawler
+	crawler Crawler
+
 	// Data Store
 	db Store
 
@@ -345,6 +348,12 @@ func NewServer(bind string, options ...Option) (*Server, error) {
 		return nil, fmt.Errorf("error validating config: %w", err)
 	}
 
+	crawler, err := NewCrawler()
+	if err != nil {
+		log.WithError(err).Error("error creating crawler")
+		return nil, err
+	}
+
 	db, err := NewStore(config.Store)
 	if err != nil {
 		log.WithError(err).Error("error creating store")
@@ -406,6 +415,9 @@ func NewServer(bind string, options ...Option) (*Server, error) {
 		// API
 		api: api,
 
+		// Crawler
+		crawler: crawler,
+
 		// Data Store
 		db: db,
 
@@ -430,6 +442,9 @@ func NewServer(bind string, options ...Option) (*Server, error) {
 	server.cron.Start()
 	log.Info("started background jobs")
 
+	server.crawler.Start()
+	log.Infof("started crawler")
+
 	server.setupMetrics()
 	log.Infof("serving metrics endpoint at %s/metrics", server.config.BaseURL)