shinyoukai/spyda
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replaced all other referenced to twt/twtxt

Browse Source
This commit is contained in:
James Mills
2021-01-30 15:06:58 +10:00
parent e0501407b5
commit 0454b9b62b
14 changed files with 138 additions and 1728 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
internal/config.go
View File

@@ -7,13 +7,11 @@ import (
"math/rand"
"net/url"
"os"
"regexp"
"strings"
"time"
"github.com/gabstv/merger"
"github.com/goccy/go-yaml"
"git.mills.io/prologic/spyda/types"
log "github.com/sirupsen/logrus"
)
@@ -25,77 +23,50 @@ var (
type Settings struct {
Name string `yaml:"pod_name"`
Description string `yaml:"pod_description"`
MaxTwtLength int `yaml:"max_twt_length"`
OpenProfiles bool `yaml:"open_profiles"`
OpenRegistrations bool `yaml:"open_registrations"`
}
// Config contains the server configuration parameters
type Config struct {
Debug bool
Data string
Name string
Description string
Store string
Theme string
BaseURL string
AdminUser string
AdminName string
AdminEmail string
FeedSources []string
RegisterMessage string
CookieSecret string
TwtPrompts []string
TwtsPerPage int
MaxUploadSize int64
MaxTwtLength int
MaxCacheTTL time.Duration
MaxCacheItems int
MsgsPerPage int
OpenProfiles bool
OpenRegistrations bool
SessionExpiry time.Duration
SessionCacheTTL time.Duration
TranscoderTimeout time.Duration
Data string
Name string
Description string
Store string
Theme string
BaseURL string
AdminUser string
AdminPass string
AdminName string
AdminEmail string
SearchPrompts []string
ResultsPerPage int
APISessionTime time.Duration
SessionExpiry time.Duration
SessionCacheTTL time.Duration
APISigningKey string
CookieSecret string
MagicLinkSecret string
SMTPBind string
POP3Bind string
SMTPHost string
SMTPPort int
SMTPUser string
SMTPPass string
SMTPFrom string
MaxFetchLimit int64
APISessionTime time.Duration
APISigningKey string
baseURL *url.URL
whitelistedDomains []*regexp.Regexp
WhitelistedDomains []string
// path string
}
var _ types.FmtOpts = (*Config)(nil)
func (c *Config) IsLocalURL(url string) bool {
if NormalizeURL(url) == "" {
return false
}
return strings.HasPrefix(NormalizeURL(url), NormalizeURL(c.BaseURL))
}
func (c *Config) LocalURL() *url.URL { return c.baseURL }
func (c *Config) ExternalURL(nick, uri string) string { return URLForExternalProfile(c, nick, uri) }
func (c *Config) UserURL(url string) string { return UserURL(url) }
func (c *Config) LocalURL() *url.URL { return c.baseURL }
// Settings returns a `Settings` struct containing pod settings that can
// then be persisted to disk to override some configuration options.
@@ -109,28 +80,10 @@ func (c *Config) Settings() *Settings {
return settings
}
// WhitelistedDomain returns true if the domain provided is a whiltelisted
// domain as per the configuration
func (c *Config) WhitelistedDomain(domain string) (bool, bool) {
// Always per mit our own domain
ourDomain := strings.TrimPrefix(strings.ToLower(c.baseURL.Hostname()), "www.")
if domain == ourDomain {
return true, true
}
// Check against list of whitelistedDomains (regexes)
for _, re := range c.whitelistedDomains {
if re.MatchString(domain) {
return true, false
}
}
return false, false
}
// RandomTwtPrompt returns a random Twt Prompt for display by the UI
func (c *Config) RandomTwtPrompt() string {
n := rand.Int() % len(c.TwtPrompts)
return c.TwtPrompts[n]
// RandomSearchPrompt returns a random Search Prompt for display by the UI
func (c *Config) RandomSearchPrompt() string {
n := rand.Int() % len(c.SearchPrompts)
return c.SearchPrompts[n]
}
// Validate validates the configuration is valid which for the most part

63
internal/context.go
View File

@@ -1,7 +1,6 @@
package internal
import (
"fmt"
"html/template"
"net/http"
"strings"
@@ -13,7 +12,6 @@ import (
"git.mills.io/prologic/spyda/internal/session"
"git.mills.io/prologic/spyda/types"
"github.com/justinas/nosurf"
"github.com/theplant-retired/timezones"
)
type Meta struct {
@@ -31,24 +29,15 @@ type Context struct {
Debug bool
BaseURL string
InstanceName string
SoftwareVersion string
TwtsPerPage int
TwtPrompt string
MaxTwtLength int
RegisterDisabled bool
OpenProfiles bool
RegisterDisabledMessage string
BaseURL string
InstanceName string
SoftwareVersion string
Timezones []*timezones.Zoneinfo
SearchPrompt string
Reply string
Username string
User *User
Tokens []*Token
LastTwt types.Twt
Profile types.Profile
Authenticated bool
IsAdmin bool
@@ -65,20 +54,10 @@ type Context struct {
Links types.Links
Alternatives types.Alternatives
Messages Messages
NewMessages int
Twter types.Twter
Twts types.Twts
BlogPost *BlogPost
BlogPosts BlogPosts
Feeds []*Feed
FeedSources FeedSourceMap
Pager *paginator.Paginator
Pager *paginator.Paginator
// Report abuse
ReportNick string
ReportURL string
ReportURL string
// Reset Password Token
PasswordResetToken string
@@ -91,21 +70,15 @@ func NewContext(conf *Config, db Store, req *http.Request) *Context {
ctx := &Context{
Debug: conf.Debug,
BaseURL: conf.BaseURL,
InstanceName: conf.Name,
SoftwareVersion: spyda.FullVersion(),
TwtsPerPage: conf.TwtsPerPage,
TwtPrompt: conf.RandomTwtPrompt(),
MaxTwtLength: conf.MaxTwtLength,
RegisterDisabled: !conf.OpenRegistrations,
OpenProfiles: conf.OpenProfiles,
LastTwt: types.NilTwt,
BaseURL: conf.BaseURL,
InstanceName: conf.Name,
SoftwareVersion: spyda.FullVersion(),
SearchPrompt: conf.RandomSearchPrompt(),
Commit: spyda.Commit,
Theme: conf.Theme,
Timezones: timezones.AllZones,
Title: "",
Meta: Meta{
Title: DefaultMetaTitle,
@@ -113,14 +86,6 @@ func NewContext(conf *Config, db Store, req *http.Request) *Context {
Keywords: DefaultMetaKeywords,
Description: conf.Description,
},
Alternatives: types.Alternatives{
types.Alternative{
Type: "application/atom+xml",
Title: fmt.Sprintf("%s local feed", conf.Name),
URL: fmt.Sprintf("%s/atom.xml", conf.BaseURL),
},
},
}
ctx.CSRFToken = nosurf.Token(req)
@@ -138,11 +103,6 @@ func NewContext(conf *Config, db Store, req *http.Request) *Context {
log.WithError(err).Warnf("error loading user object for %s", ctx.Username)
}
ctx.Twter = types.Twter{
Nick: user.Username,
URL: URLForUser(conf, user.Username),
}
ctx.User = user
tokens, err := db.GetUserTokens(user)
@@ -153,7 +113,6 @@ func NewContext(conf *Config, db Store, req *http.Request) *Context {
} else {
ctx.User = &User{}
ctx.Twter = types.Twter{}
}
if ctx.Username == conf.AdminUser {

449
internal/models.go
View File

@@ -2,72 +2,22 @@ package internal
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"github.com/creasty/defaults"
"git.mills.io/prologic/spyda/types"
log "github.com/sirupsen/logrus"
)
const (
maxUserFeeds = 5 // 5 is < 7 and humans can only really handle ~7 things
)
var (
ErrFeedAlreadyExists = errors.New("error: feed already exists by that name")
ErrAlreadyFollows = errors.New("error: you already follow this feed")
ErrTooManyFeeds = errors.New("error: you have too many feeds")
)
// Feed ...
type Feed struct {
Name string
Description string
URL string
CreatedAt time.Time
Followers map[string]string `default:"{}"`
remotes map[string]string
}
// User ...
type User struct {
Username string
Password string
Tagline string
Email string // DEPRECATED: In favor of storing a Hashed Email
URL string
Email string
CreatedAt time.Time
Theme string `default:"auto"`
Recovery string `default:"auto"`
DisplayDatesInTimezone string `default:"UTC"`
IsFollowersPubliclyVisible bool `default:"true"`
IsFollowingPubliclyVisible bool `default:"true"`
IsBookmarksPubliclyVisible bool `default:"true"`
Feeds []string `default:"[]"`
Tokens []string `default:"[]"`
SMTPToken string `default:""`
POP3Token string `default:""`
Bookmarks map[string]string `default:"{}"`
Followers map[string]string `default:"{}"`
Following map[string]string `default:"{}"`
Muted map[string]string `default:"{}"`
muted map[string]string
remotes map[string]string
sources map[string]string
}
// Token ...
@@ -100,124 +50,6 @@ func (t *Token) Bytes() ([]byte, error) {
return data, nil
}
func CreateFeed(conf *Config, db Store, user *User, name string, force bool) error {
if user != nil {
if !force && len(user.Feeds) > maxUserFeeds {
return ErrTooManyFeeds
}
}
fn := filepath.Join(conf.Data, feedsDir, name)
stat, err := os.Stat(fn)
if err == nil && !force {
return ErrFeedAlreadyExists
}
if stat == nil {
if err := ioutil.WriteFile(fn, []byte{}, 0644); err != nil {
return err
}
}
if user != nil {
if !user.OwnsFeed(name) {
user.Feeds = append(user.Feeds, name)
}
}
followers := make(map[string]string)
if user != nil {
followers[user.Username] = user.URL
}
feed := NewFeed()
feed.Name = name
feed.URL = URLForUser(conf, name)
feed.Followers = followers
feed.CreatedAt = time.Now()
if err := db.SetFeed(name, feed); err != nil {
return err
}
if user != nil {
user.Follow(name, feed.URL)
}
return nil
}
func DetachFeedFromOwner(db Store, user *User, feed *Feed) (err error) {
delete(user.Following, feed.Name)
delete(user.sources, feed.URL)
user.Feeds = RemoveString(user.Feeds, feed.Name)
if err = db.SetUser(user.Username, user); err != nil {
return
}
delete(feed.Followers, user.Username)
if err = db.SetFeed(feed.Name, feed); err != nil {
return
}
return nil
}
func RemoveFeedOwnership(db Store, user *User, feed *Feed) (err error) {
user.Feeds = RemoveString(user.Feeds, feed.Name)
if err = db.SetUser(user.Username, user); err != nil {
return
}
return nil
}
func AddFeedOwnership(db Store, user *User, feed *Feed) (err error) {
user.Feeds = append(user.Feeds, feed.Name)
if err = db.SetUser(user.Username, user); err != nil {
return
}
return nil
}
// NewFeed ...
func NewFeed() *Feed {
feed := &Feed{}
if err := defaults.Set(feed); err != nil {
log.WithError(err).Error("error creating new feed object")
}
return feed
}
// LoadFeed ...
func LoadFeed(data []byte) (feed *Feed, err error) {
feed = &Feed{}
if err := defaults.Set(feed); err != nil {
return nil, err
}
if err = json.Unmarshal(data, &feed); err != nil {
return nil, err
}
if feed.Followers == nil {
feed.Followers = make(map[string]string)
}
feed.remotes = make(map[string]string)
for n, u := range feed.Followers {
if u = NormalizeURL(u); u == "" {
continue
}
feed.remotes[u] = n
}
return
}
// NewUser ...
func NewUser() *User {
user := &User{}
@@ -237,104 +69,9 @@ func LoadUser(data []byte) (user *User, err error) {
return nil, err
}
if user.SMTPToken == "" {
user.SMTPToken = GenerateRandomToken()
}
if user.POP3Token == "" {
user.POP3Token = GenerateRandomToken()
}
if user.Bookmarks == nil {
user.Bookmarks = make(map[string]string)
}
if user.Followers == nil {
user.Followers = make(map[string]string)
}
if user.Following == nil {
user.Following = make(map[string]string)
}
user.muted = make(map[string]string)
for n, u := range user.Muted {
if u = NormalizeURL(u); u == "" {
continue
}
user.muted[u] = n
}
user.remotes = make(map[string]string)
for n, u := range user.Followers {
if u = NormalizeURL(u); u == "" {
continue
}
user.remotes[u] = n
}
user.sources = make(map[string]string)
for n, u := range user.Following {
if u = NormalizeURL(u); u == "" {
continue
}
user.sources[u] = n
}
return
}
func (f *Feed) AddFollower(nick, url string) {
url = NormalizeURL(url)
f.Followers[nick] = url
f.remotes[url] = nick
}
func (f *Feed) FollowedBy(url string) bool {
_, ok := f.remotes[NormalizeURL(url)]
return ok
}
func (f *Feed) Source() types.Feeds {
feeds := make(types.Feeds)
feeds[types.Feed{Nick: f.Name, URL: f.URL}] = true
return feeds
}
func (f *Feed) Profile(baseURL string, viewer *User) types.Profile {
var (
follows bool
followedBy bool
muted bool
)
if viewer != nil {
follows = viewer.Follows(f.URL)
followedBy = viewer.FollowedBy(f.URL)
muted = viewer.HasMuted(f.URL)
}
return types.Profile{
Type: "Feed",
Username: f.Name,
Tagline: f.Description,
URL: f.URL,
BlogsURL: URLForBlogs(baseURL, f.Name),
Follows: follows,
FollowedBy: followedBy,
Muted: muted,
Followers: f.Followers,
}
}
func (f *Feed) Bytes() ([]byte, error) {
data, err := json.Marshal(f)
if err != nil {
return nil, err
}
return data, nil
}
func (u *User) String() string {
url, err := url.Parse(u.URL)
if err != nil {
@@ -361,190 +98,6 @@ func (u *User) HasToken(token string) bool {
return false
}
func (u *User) OwnsFeed(name string) bool {
name = NormalizeFeedName(name)
for _, feed := range u.Feeds {
if NormalizeFeedName(feed) == name {
return true
}
}
return false
}
func (u *User) Is(url string) bool {
if NormalizeURL(url) == "" {
return false
}
return u.URL == NormalizeURL(url)
}
func (u *User) Bookmark(hash string) {
if _, ok := u.Bookmarks[hash]; !ok {
u.Bookmarks[hash] = ""
} else {
delete(u.Bookmarks, hash)
}
}
func (u *User) Bookmarked(hash string) bool {
_, ok := u.Bookmarks[hash]
return ok
}
func (u *User) AddFollower(nick, url string) {
url = NormalizeURL(url)
u.Followers[nick] = url
u.remotes[url] = nick
}
func (u *User) FollowedBy(url string) bool {
_, ok := u.remotes[NormalizeURL(url)]
return ok
}
func (u *User) Mute(nick, url string) {
if !u.HasMuted(url) {
u.Muted[nick] = url
u.muted[url] = nick
}
}
func (u *User) Unmute(nick string) {
url, ok := u.Muted[nick]
if ok {
delete(u.Muted, nick)
delete(u.muted, url)
}
}
func (u *User) Follow(nick, url string) {
if !u.Follows(url) {
u.Following[nick] = url
u.sources[url] = nick
}
}
func (u *User) FollowAndValidate(conf *Config, nick, url string) error {
if err := ValidateFeed(conf, nick, url); err != nil {
return err
}
if u.Follows(url) {
return ErrAlreadyFollows
}
u.Following[nick] = url
u.sources[url] = nick
return nil
}
func (u *User) Follows(url string) bool {
_, ok := u.sources[NormalizeURL(url)]
return ok
}
func (u *User) HasMuted(url string) bool {
_, ok := u.muted[NormalizeURL(url)]
return ok
}
func (u *User) Source() types.Feeds {
feeds := make(types.Feeds)
feeds[types.Feed{Nick: u.Username, URL: u.URL}] = true
return feeds
}
func (u *User) Sources() types.Feeds {
// Ensure we fetch the user's own posts in the cache
feeds := u.Source()
for url, nick := range u.sources {
feeds[types.Feed{Nick: nick, URL: url}] = true
}
return feeds
}
func (u *User) Profile(baseURL string, viewer *User) types.Profile {
var (
follows bool
followedBy bool
muted bool
)
if viewer != nil {
if viewer.Is(u.URL) {
follows = true
followedBy = true
} else {
follows = viewer.Follows(u.URL)
followedBy = viewer.FollowedBy(u.URL)
}
muted = viewer.HasMuted(u.URL)
}
return types.Profile{
Type: "User",
Username: u.Username,
Tagline: u.Tagline,
URL: u.URL,
BlogsURL: URLForBlogs(baseURL, u.Username),
Follows: follows,
FollowedBy: followedBy,
Muted: muted,
Followers: u.Followers,
Following: u.Following,
Bookmarks: u.Bookmarks,
}
}
func (u *User) Twter() types.Twter {
return types.Twter{Nick: u.Username, URL: u.URL}
}
func (u *User) Filter(twts []types.Twt) (filtered []types.Twt) {
// fast-path
if len(u.muted) == 0 {
return twts
}
for _, twt := range twts {
if u.HasMuted(twt.Twter().URL) {
continue
}
filtered = append(filtered, twt)
}
return
}
func (u *User) Reply(twt types.Twt) string {
mentionsSet := make(map[string]bool)
for _, m := range twt.Mentions() {
twter := m.Twter()
if _, ok := mentionsSet[twter.Nick]; !ok && twter.Nick != u.Username {
mentionsSet[twter.Nick] = true
}
}
mentions := []string{fmt.Sprintf("@%s", twt.Twter().Nick)}
for nick := range mentionsSet {
mentions = append(mentions, fmt.Sprintf("@%s", nick))
}
mentions = UniqStrings(mentions)
subject := twt.Subject()
if subject != "" {
subject = FormatMentionsAndTagsForSubject(subject)
return fmt.Sprintf("%s %s ", strings.Join(mentions, " "), subject)
}
return fmt.Sprintf("%s ", strings.Join(mentions, " "))
}
func (u *User) Bytes() ([]byte, error) {
data, err := json.Marshal(u)
if err != nil {

129
internal/options.go
View File

@@ -26,8 +26,9 @@ const (
// DefaultAdminXXX is the default admin user / pod operator
DefaultAdminUser = "admin"
DefaultAdminPass = "admiN"
DefaultAdminName = "Administrator"
DefaultAdminEmail = "support@twt.social"
DefaultAdminEmail = "support@spyda.dev"
// DefaultName is the default instance name
DefaultName = "spyda.dev"
@@ -41,36 +42,11 @@ const (
// DefaultTheme is the default theme to use ('light' or 'dark')
DefaultTheme = "dark"
// DefaultOpenRegistrations is the default for open user registrations
DefaultOpenRegistrations = false
// DefaultRegisterMessage is the default message displayed when registrations are disabled
DefaultRegisterMessage = ""
// DefaultCookieSecret is the server's default cookie secret
DefaultCookieSecret = InvalidConfigValue
// DefaultTwtsPerPage is the server's default twts per page to display
DefaultTwtsPerPage = 50
// DefaultMaxTwtLength is the default maximum length of posts permitted
DefaultMaxTwtLength = 288
// DefaultMaxCacheTTL is the default maximum cache ttl of twts in memory
DefaultMaxCacheTTL = time.Hour * 24 * 10 // 10 days 28 days 28 days 28 days
// DefaultMaxCacheItems is the default maximum cache items (per feed source)
// of twts in memory
DefaultMaxCacheItems = DefaultTwtsPerPage * 3 // We get bored after paging thorughh > 3 pages :D
// DefaultMsgPerPage is the server's default msgs per page to display
DefaultMsgsPerPage = 20
// DefaultOpenProfiles is the default for whether or not to have open user profiles
DefaultOpenProfiles = false
// DefaultMaxUploadSize is the default maximum upload size permitted
DefaultMaxUploadSize = 1 << 24 // ~16MB (enough for high-res photos)
// DefaultResultsPerPage is the server's default results per page to display
DefaultResultsPerPage = 10
// DefaultSessionCacheTTL is the server's default session cache ttl
DefaultSessionCacheTTL = 1 * time.Hour
@@ -78,16 +54,9 @@ const (
// DefaultSessionExpiry is the server's default session expiry time
DefaultSessionExpiry = 240 * time.Hour // 10 days
// DefaultTranscoderTimeout is the default vodeo transcoding timeout
DefaultTranscoderTimeout = 10 * time.Minute // 10mins
// DefaultMagicLinkSecret is the jwt magic link secret
DefaultMagicLinkSecret = InvalidConfigValue
// Default Messaging settings
DefaultSMTPBind = "0.0.0.0:8025"
DefaultPOP3Bind = "0.0.0.0:8110"
// Default SMTP configuration
DefaultSMTPHost = "smtp.gmail.com"
DefaultSMTPPort = 587
@@ -95,9 +64,6 @@ const (
DefaultSMTPPass = InvalidConfigValue
DefaultSMTPFrom = InvalidConfigValue
// DefaultMaxFetchLimit is the maximum fetch fetch limit in bytes
DefaultMaxFetchLimit = 1 << 21 // ~2MB (or more than enough for a year)
// DefaultAPISessionTime is the server's default session time for API tokens
DefaultAPISessionTime = 240 * time.Hour // 10 days
@@ -121,27 +87,26 @@ func NewConfig() *Config {
return &Config{
Debug: DefaultDebug,
Name: DefaultName,
Description: DefaultMetaDescription,
Store: DefaultStore,
Theme: DefaultTheme,
BaseURL: DefaultBaseURL,
AdminUser: DefaultAdminUser,
FeedSources: DefaultFeedSources,
RegisterMessage: DefaultRegisterMessage,
CookieSecret: DefaultCookieSecret,
TwtPrompts: DefaultTwtPrompts,
TwtsPerPage: DefaultTwtsPerPage,
MaxTwtLength: DefaultMaxTwtLength,
MsgsPerPage: DefaultMsgsPerPage,
OpenProfiles: DefaultOpenProfiles,
OpenRegistrations: DefaultOpenRegistrations,
SessionExpiry: DefaultSessionExpiry,
MagicLinkSecret: DefaultMagicLinkSecret,
SMTPHost: DefaultSMTPHost,
SMTPPort: DefaultSMTPPort,
SMTPUser: DefaultSMTPUser,
SMTPPass: DefaultSMTPPass,
Name: DefaultName,
Description: DefaultMetaDescription,
Store: DefaultStore,
Theme: DefaultTheme,
BaseURL: DefaultBaseURL,
AdminUser: DefaultAdminUser,
AdminPass: DefaultAdminPass,
CookieSecret: DefaultCookieSecret,
MagicLinkSecret: DefaultMagicLinkSecret,
SearchPrompts: DefaultSearchPrompts,
ResultsPerPage: DefaultResultsPerPage,
SessionExpiry: DefaultSessionExpiry,
SMTPHost: DefaultSMTPHost,
SMTPPort: DefaultSMTPPort,
SMTPUser: DefaultSMTPUser,
SMTPPass: DefaultSMTPPass,
}
}
@@ -257,50 +222,10 @@ func WithCookieSecret(secret string) Option {
}
}
// WithTwtsPerPage sets the server's twts per page
func WithTwtsPerPage(twtsPerPage int) Option {
// WithResultsPerPage sets the server's results per page
func WithResultsPerPage(resultsPerPage int) Option {
return func(cfg *Config) error {
cfg.TwtsPerPage = twtsPerPage
return nil
}
}
// WithMaxTwtLength sets the maximum length of posts permitted on the server
func WithMaxTwtLength(maxTwtLength int) Option {
return func(cfg *Config) error {
cfg.MaxTwtLength = maxTwtLength
return nil
}
}
// WithMaxCacheTTL sets the maximum cache ttl of twts in memory
func WithMaxCacheTTL(maxCacheTTL time.Duration) Option {
return func(cfg *Config) error {
cfg.MaxCacheTTL = maxCacheTTL
return nil
}
}
// WithMaxCacheItems sets the maximum cache items (per feed source) of twts in memory
func WithMaxCacheItems(maxCacheItems int) Option {
return func(cfg *Config) error {
cfg.MaxCacheItems = maxCacheItems
return nil
}
}
// WithOpenProfiles sets whether or not to have open user profiles
func WithOpenProfiles(openProfiles bool) Option {
return func(cfg *Config) error {
cfg.OpenProfiles = openProfiles
return nil
}
}
// WithMaxUploadSize sets the maximum upload size permitted by the server
func WithMaxUploadSize(maxUploadSize int64) Option {
return func(cfg *Config) error {
cfg.MaxUploadSize = maxUploadSize
cfg.ResultsPerPage = resultsPerPage
return nil
}
}

38
internal/pages/abuse.md Normal file
View File

@@ -0,0 +1,38 @@
---
title: Spyda Abuse Policy
---
# Abuse Policy
> Help us build and maintain a quality search experience.
At Spyda.search, we believe that the Web should be freely searchable without
having to track you or collect data about your searches or behavior. Everyone
should be able to benefit from an open search experience without worrying about
privacy.
That being said there are a basic set of guidelines that help ensure that
search is of the highest quality and safe to use and considered acceptable
by everyone from all backgrounds and walks of live.
Therefore the following outlines the types of content that Spyda will not
permit and if found will be immediately deleted and potenaitlly blacklisted
from future indexing.
**If you have a website or know of a website that has been crawled and indexed
that violates these guidelines, please contact us immediately at /support so
we can remove the offending data.**
- **Hate speech:** Any website found to be hosting content that attacks other people on the basis of race, ethnicity, national origin, sexual orientation, gender, gender identity, religious affiliation, age, disability, or disease. We will not index websites whose primary purpose is inciting harm towards others on the basis of these categories.
- **Threats of violence:** Any websites found to be treating, planning or otherwise of a treating nature towards others to organize, promote, or incite acts of real-world violence or terrorism.
- **Illegal activities**: Any websites found to be hosting content for unlawful purposes or in furtherance of illegal activities. International webmasters agree to comply with all local laws regarding online conduct and acceptable content.
## What Happens If a Website Violates These Guidelines
- It will be immediately removed from our indexes.
- We _may_ attempt to reach the webmaster to correct the situation.
- We _may_ blacklist the website from future indexing.
Thanks for helping us build and maintain a safe and quality search experience for all!
We reserve the right to update these guidelines as needed.

348
internal/rice-box.go
View File

File diff suppressed because one or more lines are too long

9
internal/robots.go
View File

@@ -11,18 +11,11 @@ import (
const robotsTpl = `User-Agent: *
Disallow: /
Allow: /
Allow: /twt
Allow: /user
Allow: /feed
Allow: /about
Allow: /abuse
Allow: /help
Allow: /blogs
Allow: /privacy
Allow: /support
Allow: /search
Allow: /external
Allow: /atom.xml
Allow: /media
`
// RobotsHandler ...

175
internal/server.go
View File

@@ -12,7 +12,6 @@ import (
rice "github.com/GeertJohan/go.rice"
"github.com/NYTimes/gziphandler"
humanize "github.com/dustin/go-humanize"
"github.com/gabstv/merger"
"github.com/justinas/nosurf"
"github.com/prologic/observe"
@@ -33,7 +32,7 @@ var (
)
func init() {
metrics = observe.NewMetrics("twtd")
metrics = observe.NewMetrics("spyda")
}
// Server ...
@@ -234,45 +233,6 @@ func (s *Server) setupMetrics() {
"Number of unique feeds in the global feed cache",
)
// feed cache size
metrics.NewGauge(
"cache", "twts",
"Number of active twts in the global feed cache",
)
// blogs cache size
metrics.NewGaugeFunc(
"cache", "blogs",
"Number of blogs in the blogs cache",
func() float64 {
return float64(s.blogs.Count())
},
)
// feed cache processing time
metrics.NewGauge(
"cache", "last_processed_seconds",
"Number of seconds for a feed cache cycle",
)
// feed cache limited fetch (feed exceeded MaxFetchLImit or unknown size)
metrics.NewCounter(
"cache", "limited",
"Number of feed cache fetches affected by MaxFetchLimit",
)
// archive size
metrics.NewCounter(
"archive", "size",
"Number of items inserted into the global feed archive",
)
// archive errors
metrics.NewCounter(
"archive", "error",
"Number of items errored inserting into the global feed archive",
)
// server info
metrics.NewGaugeVec(
"server", "info",
@@ -353,140 +313,40 @@ func (s *Server) initRoutes() {
s.router.GET("/privacy", s.PageHandler("privacy"))
s.router.GET("/abuse", s.PageHandler("abuse"))
s.router.GET("/", s.TimelineHandler())
s.router.HEAD("/", s.TimelineHandler())
s.router.GET("/", s.IndexHandler())
s.router.HEAD("/", s.IndexHandler())
s.router.GET("/robots.txt", s.RobotsHandler())
s.router.HEAD("/robots.txt", s.RobotsHandler())
s.router.GET("/discover", s.am.MustAuth(s.DiscoverHandler()))
s.router.GET("/mentions", s.am.MustAuth(s.MentionsHandler()))
s.router.GET("/search", s.SearchHandler())
s.router.HEAD("/twt/:hash", s.PermalinkHandler())
s.router.GET("/twt/:hash", s.PermalinkHandler())
s.router.GET("/bookmark/:hash", s.BookmarkHandler())
s.router.POST("/bookmark/:hash", s.BookmarkHandler())
s.router.HEAD("/conv/:hash", s.ConversationHandler())
s.router.GET("/conv/:hash", s.ConversationHandler())
s.router.GET("/feeds", s.am.MustAuth(s.FeedsHandler()))
s.router.POST("/feed", s.am.MustAuth(s.FeedHandler()))
s.router.POST("/post", s.am.MustAuth(s.PostHandler()))
s.router.PATCH("/post", s.am.MustAuth(s.PostHandler()))
s.router.DELETE("/post", s.am.MustAuth(s.PostHandler()))
// Private Messages
s.router.GET("/messages", s.am.MustAuth(s.ListMessagesHandler()))
s.router.GET("/messages/:msgid", s.am.MustAuth(s.ViewMessageHandler()))
s.router.POST("/messages/send", s.am.MustAuth(s.SendMessageHandler()))
s.router.POST("/messages/delete", s.am.MustAuth(s.DeleteMessagesHandler()))
s.router.POST("/blog", s.am.MustAuth(s.CreateOrUpdateBlogHandler()))
s.router.GET("/blogs/:author", s.ListBlogsHandler())
s.router.GET("/blog/:author/:year/:month/:date/:slug", s.ViewBlogHandler())
s.router.HEAD("/blog/:author/:year/:month/:date/:slug", s.ViewBlogHandler())
s.router.GET("/blog/:author/:year/:month/:date/:slug/edit", s.EditBlogHandler())
s.router.GET("/blog/:author/:year/:month/:date/:slug/delete", s.DeleteBlogHandler())
s.router.GET("/blog/:author/:year/:month/:date/:slug/publish", s.PublishBlogHandler())
if s.config.OpenProfiles {
s.router.GET("/user/:nick/", s.ProfileHandler())
s.router.GET("/user/:nick/config.yaml", s.UserConfigHandler())
} else {
s.router.GET("/user/:nick/", s.am.MustAuth(s.ProfileHandler()))
s.router.GET("/user/:nick/config.yaml", s.am.MustAuth(s.UserConfigHandler()))
}
s.router.GET("/user/:nick/avatar", s.AvatarHandler())
s.router.HEAD("/user/:nick/avatar", s.AvatarHandler())
s.router.GET("/user/:nick/followers", s.FollowersHandler())
s.router.GET("/user/:nick/following", s.FollowingHandler())
s.router.GET("/user/:nick/bookmarks", s.BookmarksHandler())
s.router.GET("/pod/avatar", s.PodAvatarHandler())
// WebMentions
s.router.POST("/user/:nick/webmention", s.WebMentionHandler())
// External Feeds
s.router.GET("/external", s.ExternalHandler())
s.router.GET("/externalAvatar", s.ExternalAvatarHandler())
s.router.HEAD("/externalAvatar", s.ExternalAvatarHandler())
// External Queries (protected by a short-lived token)
s.router.GET("/whoFollows", s.WhoFollowsHandler())
// Syndication Formats (RSS, Atom, JSON Feed)
s.router.HEAD("/atom.xml", s.SyndicationHandler())
s.router.HEAD("/user/:nick/atom.xml", s.SyndicationHandler())
s.router.GET("/atom.xml", s.SyndicationHandler())
s.router.GET("/user/:nick/atom.xml", s.SyndicationHandler())
s.router.GET("/feed/:name/manage", s.am.MustAuth(s.ManageFeedHandler()))
s.router.POST("/feed/:name/manage", s.am.MustAuth(s.ManageFeedHandler()))
s.router.POST("/feed/:name/archive", s.am.MustAuth(s.ArchiveFeedHandler()))
s.router.GET("/login", s.am.HasAuth(s.LoginHandler()))
s.router.POST("/login", s.LoginHandler())
s.router.GET("/logout", s.LogoutHandler())
s.router.POST("/logout", s.LogoutHandler())
s.router.GET("/register", s.am.HasAuth(s.RegisterHandler()))
s.router.POST("/register", s.RegisterHandler())
// Reset Password
s.router.GET("/resetPassword", s.ResetPasswordHandler())
s.router.POST("/resetPassword", s.ResetPasswordHandler())
s.router.GET("/newPassword", s.ResetPasswordMagicLinkHandler())
s.router.POST("/newPassword", s.NewPasswordHandler())
s.router.GET("/pwreset", s.ResetPasswordHandler())
s.router.POST("/pwreset", s.ResetPasswordHandler())
s.router.GET("/chpasswd", s.ResetPasswordMagicLinkHandler())
s.router.POST("/chpasswd", s.NewPasswordHandler())
// Media Handling
s.router.GET("/media/:name", s.MediaHandler())
s.router.HEAD("/media/:name", s.MediaHandler())
s.router.POST("/upload", s.am.MustAuth(s.UploadMediaHandler()))
// Task State
s.router.GET("/task/:uuid", s.TaskHandler())
// User/Feed Lookups
s.router.GET("/lookup", s.am.MustAuth(s.LookupHandler()))
s.router.GET("/follow", s.am.MustAuth(s.FollowHandler()))
s.router.POST("/follow", s.am.MustAuth(s.FollowHandler()))
s.router.GET("/import", s.am.MustAuth(s.ImportHandler()))
s.router.POST("/import", s.am.MustAuth(s.ImportHandler()))
s.router.GET("/unfollow", s.am.MustAuth(s.UnfollowHandler()))
s.router.POST("/unfollow", s.am.MustAuth(s.UnfollowHandler()))
s.router.GET("/mute", s.am.MustAuth(s.MuteHandler()))
s.router.POST("/mute", s.am.MustAuth(s.MuteHandler()))
s.router.GET("/unmute", s.am.MustAuth(s.UnmuteHandler()))
s.router.POST("/unmute", s.am.MustAuth(s.UnmuteHandler()))
s.router.GET("/transferFeed/:name", s.TransferFeedHandler())
s.router.GET("/transferFeed/:name/:transferTo", s.TransferFeedHandler())
s.router.GET("/add", s.AddHandler())
s.router.POST("/add", s.AddHandler())
s.router.GET("/settings", s.am.MustAuth(s.SettingsHandler()))
s.router.POST("/settings", s.am.MustAuth(s.SettingsHandler()))
s.router.POST("/token/delete/:signature", s.am.MustAuth(s.DeleteTokenHandler()))
s.router.GET("/config", s.am.MustAuth(s.PodConfigHandler()))
s.router.GET("/manage/pod", s.ManagePodHandler())
s.router.POST("/manage/pod", s.ManagePodHandler())
s.router.GET("/manage", s.ManageHandler())
s.router.POST("/manage", s.ManageHandler())
s.router.GET("/manage/users", s.ManageUsersHandler())
s.router.POST("/manage/adduser", s.AddUserHandler())
s.router.POST("/manage/deluser", s.DelUserHandler())
s.router.GET("/deleteFeeds", s.DeleteAccountHandler())
s.router.POST("/delete", s.am.MustAuth(s.DeleteAllHandler()))
// Support / Report Abuse handlers
s.router.GET("/support", s.SupportHandler())
@@ -687,25 +547,12 @@ func NewServer(bind string, options ...Option) (*Server, error) {
log.Infof("Admin User: %s", server.config.AdminUser)
log.Infof("Admin Name: %s", server.config.AdminName)
log.Infof("Admin Email: %s", server.config.AdminEmail)
log.Infof("Max Twts per Page: %d", server.config.TwtsPerPage)
log.Infof("Max Cache TTL: %s", server.config.MaxCacheTTL)
log.Infof("Max Cache Items: %d", server.config.MaxCacheItems)
log.Infof("Maximum length of Posts: %d", server.config.MaxTwtLength)
log.Infof("Open User Profiles: %t", server.config.OpenProfiles)
log.Infof("Open Registrations: %t", server.config.OpenRegistrations)
log.Infof("SMTP Host: %s", server.config.SMTPHost)
log.Infof("SMTP Port: %d", server.config.SMTPPort)
log.Infof("SMTP User: %s", server.config.SMTPUser)
log.Infof("SMTP From: %s", server.config.SMTPFrom)
log.Infof("Max Fetch Limit: %s", humanize.Bytes(uint64(server.config.MaxFetchLimit)))
log.Infof("Max Upload Size: %s", humanize.Bytes(uint64(server.config.MaxUploadSize)))
log.Infof("API Session Time: %s", server.config.APISessionTime)
// Warn about user registration being disabled.
if !server.config.OpenRegistrations {
log.Warn("Open Registrations are disabled as per configuration (no -R/--open-registrations)")
}
server.initRoutes()
api.initRoutes()

4
internal/templates.go
View File

@@ -40,11 +40,7 @@ func NewTemplateManager(conf *Config, blogs *BlogsCache, cache *Cache) (*Templat
funcMap["hostnameFromURL"] = HostnameFromURL
funcMap["prettyURL"] = PrettyURL
funcMap["isLocalURL"] = IsLocalURLFactory(conf)
funcMap["formatTwt"] = FormatTwtFactory(conf)
funcMap["unparseTwt"] = UnparseTwtFactory(conf)
funcMap["formatForDateTime"] = FormatForDateTime
funcMap["urlForBlog"] = URLForBlogFactory(conf, blogs)
funcMap["urlForConv"] = URLForConvFactory(conf, cache)
funcMap["isAdminUser"] = IsAdminUserFactory(conf)
m := &TemplateManager{debug: conf.Debug, templates: templates, funcMap: funcMap}

2
internal/templates/_partials.html
View File

@@ -11,7 +11,7 @@
</li>
</ul>
<ul>
<li><small>Page {{ .Page }}/{{ .PageNums }} of {{ .Nums }} Twts</small></li>
<li><small>Page {{ .Page }}/{{ .PageNums }} of {{ .Nums }} Results</small></li>
</ul>
<ul>
<li>