Add a /tasks endpoing and extra metdata to crawl tasks

internal/dispatcher.go

@@ -77,6 +77,15 @@ func (d *Dispatcher) Stop() {
 	d.quit <- true
 }
 
+// Tasks returns all tasks
+func (d *Dispatcher) Tasks() map[string]TaskResult {
+	tasks := make(map[string]TaskResult)
+	for id, task := range d.taskMap {
+		tasks[id] = task.Result()
+	}
+	return tasks
+}
+
 // Lookup returns the matching `Task` given its id
 func (d *Dispatcher) Lookup(id string) (Task, bool) {
 	task, ok := d.taskMap[id]