shinyoukai/spyda
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove a bunch of unused cruft

Browse Source
This commit is contained in:
James Mills
2022-10-05 11:11:40 +10:00
parent 06c8ce3410
commit a07f747eb1
25 changed files with 31 additions and 1542 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
internal/crawler.go
View File

@@ -29,21 +29,14 @@ func NewCrawler(conf *Config, tasks *Dispatcher, db Store, indexer Indexer) (Cra
}
func (c *crawler) loop() {
for {
select {
case url, ok := <-c.queue:
if !ok {
log.Debugf("crawler shutting down...")
return
}
log.Debugf("crawling %s", url)
uuid, err := c.tasks.Dispatch(NewCrawlTask(c.conf, c.db, c.indexer, url))
if err != nil {
log.WithError(err).Error("error creating crawl task for %s", url)
} else {
taskURL := URLForTask(c.conf.BaseURL, uuid)
log.WithField("uuid", uuid).Infof("successfully created crawl task for %s: %s", url, taskURL)
}
for url := range c.queue {
log.Debugf("crawling %s", url)
uuid, err := c.tasks.Dispatch(NewCrawlTask(c.conf, c.db, c.indexer, url))
if err != nil {
log.WithError(err).Error("error creating crawl task for %s", url)
} else {
taskURL := URLForTask(c.conf.BaseURL, uuid)
log.WithField("uuid", uuid).Infof("successfully created crawl task for %s: %s", url, taskURL)
}
}
}