Add working Crawler, Scraper and Indexer

internal/server.go

@@ -41,6 +41,9 @@ type Server struct {
 	router  *Router
 	server  *http.Server
 
+	// Indexer
+	indexer Indexer
+
 	// Crawler
 	crawler Crawler
 
@@ -348,7 +351,13 @@ func NewServer(bind string, options ...Option) (*Server, error) {
 		return nil, fmt.Errorf("error validating config: %w", err)
 	}
 
-	crawler, err := NewCrawler()
+	indexer, err := NewIndexer(config)
+	if err != nil {
+		log.WithError(err).Error("error creating indexer")
+		return nil, err
+	}
+
+	crawler, err := NewCrawler(config, indexer)
 	if err != nil {
 		log.WithError(err).Error("error creating crawler")
 		return nil, err
@@ -415,6 +424,9 @@ func NewServer(bind string, options ...Option) (*Server, error) {
 		// API
 		api: api,
 
+		// Indexer
+		indexer: indexer,
+
 		// Crawler
 		crawler: crawler,