Refactored crawler to use task dispatcher

internal/server.go

@@ -53,6 +53,9 @@ type Server struct {
 	// Scheduler
 	cron *cron.Cron
 
+	// Dispatcher
+	tasks *Dispatcher
+
 	// Auth
 	am *auth.Manager
 
@@ -93,6 +96,8 @@ func (s *Server) AddShutdownHook(f func()) {
 // Shutdown ...
 func (s *Server) Shutdown(ctx context.Context) error {
 	s.cron.Stop()
+	s.tasks.Stop()
+	s.crawler.Stop()
 
 	if err := s.server.Shutdown(ctx); err != nil {
 		log.WithError(err).Error("error shutting down server")
@@ -300,6 +305,9 @@ func (s *Server) initRoutes() {
 	s.router.GET("/chpasswd", s.ResetPasswordMagicLinkHandler())
 	s.router.POST("/chpasswd", s.NewPasswordHandler())
 
+	// Task State
+	s.router.GET("/task/:uuid", s.TaskHandler())
+
 	s.router.GET("/add", s.AddHandler())
 	s.router.POST("/add", s.AddHandler())
 
@@ -362,22 +370,12 @@ func NewServer(bind string, options ...Option) (*Server, error) {
 		return nil, err
 	}
 
-	indexer, err := NewIndexer(config)
-	if err != nil {
-		log.WithError(err).Error("error creating indexer")
-		return nil, err
-	}
-
-	crawler, err := NewCrawler(config, db, indexer)
-	if err != nil {
-		log.WithError(err).Error("error creating crawler")
-		return nil, err
-	}
-
 	router := NewRouter()
 
 	am := auth.NewManager(auth.NewOptions("/login", "/register"))
 
+	tasks := NewDispatcher(10, 100) // TODO: Make this configurable?
+
 	pm := passwords.NewScryptPasswords(nil)
 
 	sc := NewSessionStore(db, config.SessionCacheTTL)
@@ -392,6 +390,18 @@ func NewServer(bind string, options ...Option) (*Server, error) {
 		sc,
 	)
 
+	indexer, err := NewIndexer(config)
+	if err != nil {
+		log.WithError(err).Error("error creating indexer")
+		return nil, err
+	}
+
+	crawler, err := NewCrawler(config, tasks, db, indexer)
+	if err != nil {
+		log.WithError(err).Error("error creating crawler")
+		return nil, err
+	}
+
 	api := NewAPI(router, config, db, pm)
 
 	csrfHandler := nosurf.New(router)
@@ -430,6 +440,9 @@ func NewServer(bind string, options ...Option) (*Server, error) {
 		// Schedular
 		cron: cron.New(),
 
+		// Dispatcher
+		tasks: tasks,
+
 		// Auth Manager
 		am: am,
 
@@ -448,6 +461,9 @@ func NewServer(bind string, options ...Option) (*Server, error) {
 	server.cron.Start()
 	log.Info("started background jobs")
 
+	server.tasks.Start()
+	log.Info("started task dispatcher")
+
 	server.crawler.Start()
 	log.Infof("started crawler")