Refactored crawler to use task dispatcher

internal/utils.go

@@ -381,6 +381,14 @@ func URLForCached(baseURL, hash string) string {
 	)
 }
 
+func URLForTask(baseURL, uuid string) string {
+	return fmt.Sprintf(
+		"%s/task/%s",
+		strings.TrimSuffix(baseURL, "/"),
+		uuid,
+	)
+}
+
 // SafeParseInt ...
 func SafeParseInt(s string, d int) int {
 	n, e := strconv.Atoi(s)