63 lines
1.1 KiB
Go
63 lines
1.1 KiB
Go
package internal
|
|
|
|
import (
|
|
log "github.com/sirupsen/logrus"
|
|
)
|
|
|
|
type Crawler interface {
|
|
Start()
|
|
Stop()
|
|
Crawl(url string) error
|
|
}
|
|
|
|
type crawler struct {
|
|
conf *Config
|
|
tasks *Dispatcher
|
|
db Store
|
|
indexer Indexer
|
|
queue chan string
|
|
}
|
|
|
|
func NewCrawler(conf *Config, tasks *Dispatcher, db Store, indexer Indexer) (Crawler, error) {
|
|
return &crawler{
|
|
conf: conf,
|
|
tasks: tasks,
|
|
db: db,
|
|
indexer: indexer,
|
|
queue: make(chan string),
|
|
}, nil
|
|
}
|
|
|
|
func (c *crawler) loop() {
|
|
for {
|
|
select {
|
|
case url, ok := <-c.queue:
|
|
if !ok {
|
|
log.Debugf("crawler shutting down...")
|
|
return
|
|
}
|
|
log.Debugf("crawling %s", url)
|
|
uuid, err := c.tasks.Dispatch(NewCrawlTask(c.conf, c.db, c.indexer, url))
|
|
if err != nil {
|
|
log.WithError(err).Error("error creating crawl task for %s", url)
|
|
} else {
|
|
taskURL := URLForTask(c.conf.BaseURL, uuid)
|
|
log.WithField("uuid", uuid).Infof("successfully created crawl task for %s: %s", url, taskURL)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *crawler) Crawl(url string) error {
|
|
c.queue <- url
|
|
return nil
|
|
}
|
|
|
|
func (c *crawler) Stop() {
|
|
close(c.queue)
|
|
}
|
|
|
|
func (c *crawler) Start() {
|
|
go c.loop()
|
|
}
|