Add crawler skeleton and add handler

go.mod

@@ -8,13 +8,17 @@ require (
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/NYTimes/gziphandler v1.1.1
+	github.com/PuerkitoBio/goquery v1.6.1 // indirect
 	github.com/andreadipersio/securecookie v0.0.0-20131119095127-e3c3b33544ec
+	github.com/antchfx/htmlquery v1.2.3 // indirect
+	github.com/antchfx/xmlquery v1.3.3 // indirect
 	github.com/creasty/defaults v1.5.1
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/dustin/go-humanize v1.0.0
 	github.com/elithrar/simple-scrypt v1.3.0
 	github.com/gabstv/merger v1.0.1
 	github.com/go-mail/mail v2.3.1+incompatible
+	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/goccy/go-yaml v1.8.6
 	github.com/gocolly/colly/v2 v2.1.0
 	github.com/gomarkdown/markdown v0.0.0-20201113031856-722100d81a8e
@@ -25,6 +29,7 @@ require (
 	github.com/james4k/fmatter v0.0.0-20150827042251-377c8ea6259d
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/justinas/nosurf v1.1.1
+	github.com/kennygrant/sanitize v1.2.4 // indirect
 	github.com/mitchellh/copystructure v1.0.0 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/prologic/bitcask v0.3.10
@@ -32,10 +37,12 @@ require (
 	github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect
 	github.com/renstrom/shortuuid v2.0.3+incompatible
 	github.com/robfig/cron v1.2.0
+	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
 	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/sirupsen/logrus v1.7.0
 	github.com/spf13/pflag v1.0.5
 	github.com/steambap/captcha v1.3.1
+	github.com/temoto/robotstxt v1.1.1 // indirect
 	github.com/unrolled/logger v0.0.0-20201216141554-31a3694fe979
 	github.com/vcraescu/go-paginator v1.0.0
 	github.com/wblakecaldwell/profiler v0.0.0-20150908040756-6111ef1313a1