Add crawler skeleton and add handler

internal/models.go

@@ -8,6 +8,10 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+const (
+	URLHashLength = 15
+)
+
 // User ...
 type User struct {
 	Username  string
@@ -28,6 +32,15 @@ type Token struct {
 	ExpiresAt time.Time
 }
 
+// URL  ...
+type URL struct {
+	URL       string
+	CrawledAt time.Time
+	ExpiresAt time.Time
+
+	hash string
+}
+
 func LoadToken(data []byte) (token *Token, err error) {
 	token = &Token{}
 	if err := defaults.Set(token); err != nil {
@@ -99,3 +112,48 @@ func (u *User) Bytes() ([]byte, error) {
 	}
 	return data, nil
 }
+
+// NewURL ...
+func NewURL() *URL {
+	u := &URL{}
+	if err := defaults.Set(u); err != nil {
+		log.WithError(err).Error("error creating new URI object")
+	}
+	return u
+}
+
+func LoadURL(data []byte) (u *URL, err error) {
+	u = &URL{}
+	if err := defaults.Set(u); err != nil {
+		return nil, err
+	}
+
+	if err = json.Unmarshal(data, &u); err != nil {
+		return nil, err
+	}
+
+	return
+}
+
+func (u *URL) Hash() string {
+	if u.hash != "" {
+		return u.hash
+	}
+
+	hash := FastHash(u.String())
+	u.hash = hash[len(hash)-URLHashLength:]
+
+	return u.hash
+}
+
+func (u *URL) String() string {
+	return u.URL
+}
+
+func (u *URL) Bytes() ([]byte, error) {
+	data, err := json.Marshal(u)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}