Add working Crawler, Scraper and Indexer

internal/entry.go

@@ -0,0 +1,56 @@
+package internal
+
+import (
+	"encoding/json"
+
+	"github.com/creasty/defaults"
+)
+
+// Entry ...
+type Entry struct {
+	URL         string
+	Title       string
+	Author      string
+	Summary     string
+	Content     string
+	HTMLContent string
+	Length      int
+
+	hash string
+}
+
+func LoadEntry(data []byte) (entry *Entry, err error) {
+	entry = &Entry{}
+	if err := defaults.Set(entry); err != nil {
+		return nil, err
+	}
+
+	if err = json.Unmarshal(data, &entry); err != nil {
+		return nil, err
+	}
+
+	return
+}
+
+func (e *Entry) String() string {
+	return e.URL
+}
+
+func (e *Entry) Hash() string {
+	if e.hash != "" {
+		return e.hash
+	}
+
+	hash := FastHash(e.String())
+	e.hash = hash[len(hash)-URLHashLength:]
+
+	return e.hash
+}
+
+func (e *Entry) Bytes() ([]byte, error) {
+	data, err := json.Marshal(e)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}