shinyoukai/mai
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make repo a monorepo

Browse Source 
Also fixed the module URLs.

git-svn-id: file:///srv/svn/repo/mai/trunk@9 e410bdd4-646f-c54f-a7ce-fffcc4f439ae
This commit is contained in:
fattalion
2022-09-21 09:01:41 +00:00
parent 897a8b37e4
commit c4d051d74f
11 changed files with 44 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
engines/engine.go Normal file
View File

@@ -0,0 +1,16 @@
package engines
type TranslationResult struct {
SourceLanguage Language
TranslatedText string
}
type TranslationEngine interface {
InternalName() string
DisplayName() string
SourceLanguages() ([]Language, error)
TargetLanguages() ([]Language, error)
Translate(text string, from Language, to Language) (TranslationResult, error)
SupportsAutodetect() bool
DetectLanguage(text string) (Language, error)
}

7
engines/go.mod Normal file
View File

@@ -0,0 +1,7 @@
module codeberg.org/SimpleWeb/SimplyTranslate/engines
go 1.16
require github.com/PuerkitoBio/goquery v1.8.0
require golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 // indirect

16
engines/go.sum Normal file
View File

@@ -0,0 +1,16 @@
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 h1:N9Vc/rorQUDes6B9CNdIxAn5jODGj2wzfrei2x4wNj4=
golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

131
engines/google.go Normal file
View File

@@ -0,0 +1,131 @@
package engines
import (
"fmt"
"net/http"
"net/url"
"github.com/PuerkitoBio/goquery"
)
type GoogleTranslateEngine struct{}
func (_ *GoogleTranslateEngine) InternalName() string { return "google" }
func (_ *GoogleTranslateEngine) DisplayName() string { return "Google" }
func (_ *GoogleTranslateEngine) getLangs(type_ string) ([]Language, error) {
var langsType string
switch type_ {
case "source":
langsType = "sl"
case "target":
langsType = "tl"
default:
panic(fmt.Errorf("getLangs was passed an invalid language type: %s", langsType))
}
requestURL, err := url.Parse("https://translate.google.com/m")
if err != nil {
// The URL is constant, so it should never fail.
panic(err)
}
query := url.Values{}
query.Add("mui", langsType)
query.Add("hl", "en-US")
requestURL.RawQuery = query.Encode()
response, err := http.Get(requestURL.String())
if err != nil {
return nil, err
}
defer response.Body.Close()
doc, err := goquery.NewDocumentFromReader(response.Body)
if err != nil {
return nil, err
}
var langs []Language
doc.Find(".language-item").Each(func(_ int, s *goquery.Selection) {
a := s.Find("a").First()
href, exists := a.Attr("href")
// Shouldn't happen, but here goes.
if !exists {
return
}
langURL, err := url.Parse(href)
if err != nil {
return
}
langCode := langURL.Query()[langsType][0]
if langCode == "auto" {
return
}
langs = append(langs, Language{Name: a.Text(), Code: langCode})
})
return langs, nil
}
func (e *GoogleTranslateEngine) SourceLanguages() ([]Language, error) {
return e.getLangs("source")
}
func (e *GoogleTranslateEngine) TargetLanguages() ([]Language, error) {
return e.getLangs("target")
}
func (_ *GoogleTranslateEngine) SupportsAutodetect() bool { return true }
func (_ *GoogleTranslateEngine) DetectLanguage(text string) (Language, error) { return Language{}, nil }
func (_ *GoogleTranslateEngine) Translate(text string, from Language, to Language) (TranslationResult, error) {
requestURL, err := url.Parse("https://translate.google.com/m")
if err != nil {
// The URL is constant, so it should never fail.
panic(err)
}
query := url.Values{}
query.Add("sl", from.Code)
query.Add("tl", to.Code)
query.Add("hl", to.Code)
query.Add("q", text)
requestURL.RawQuery = query.Encode()
response, err := http.Get(requestURL.String())
if err != nil {
return TranslationResult{}, err
}
defer response.Body.Close()
doc, err := goquery.NewDocumentFromReader(response.Body)
if err != nil {
return TranslationResult{}, err
}
return TranslationResult{
SourceLanguage: from,
TranslatedText: doc.Find(".result-container").Text(),
}, nil
}

290
engines/iciba.go Normal file
View File

@@ -0,0 +1,290 @@
package engines
import (
"crypto/md5"
"encoding/hex"
"encoding/json"
"fmt"
"net/http"
"net/url"
)
// ICIBAEngine is an engine that fetches data from https://www.iciba.com.
type ICIBAEngine struct{}
func (_ *ICIBAEngine) InternalName() string { return "iciba" }
func (_ *ICIBAEngine) DisplayName() string { return "iCIBA" }
var icibaLanguages = []Language{
// ICIBA does have an API, but they return Chinese names.
// For languages already present in Google translate, the English
// names in that engine file are used; Otherwise official names
// as researched on Wikipedia are used. They're validated against
// the Chinese names to the best of my ability.
// Missing "cni", "kbh", "tmh"
// due to conflict between ISO-639 table and Chinese label
// one "//" means on iciba but not on google
{Name: "Achinese", Code: "ace"}, //
{Name: "Achuar-Shiwiar", Code: "acu"}, //
{Name: "Afrikaans", Code: "af"},
{Name: "Aguaruna", Code: "agr"}, //
{Name: "Akawaio", Code: "ake"}, //
{Name: "Albanian", Code: "sq"},
{Name: "Amharic", Code: "am"},
{Name: "Arabic", Code: "ar"},
{Name: "Armenian", Code: "hy"},
{Name: "Azerbaijani", Code: "az"},
{Name: "Barasana-Eduria", Code: "bsn"}, //
{Name: "Bashkir", Code: "ba"}, //
{Name: "Basque", Code: "eu"},
{Name: "Belarusian", Code: "be"},
{Name: "Bemba", Code: "bem"}, //
{Name: "Bengali", Code: "bn"},
{Name: "Berber", Code: "ber"}, //
{Name: "Bislama", Code: "bi"}, //
{Name: "Bosnian", Code: "bs"},
{Name: "Breton", Code: "br"}, //
{Name: "Bulgarian", Code: "bg"},
{Name: "Cabécar", Code: "cjp"}, //
{Name: "Cantonese", Code: "yue"},
{Name: "Catalan", Code: "ca"},
{Name: "Cebuano", Code: "ceb"},
{Name: "Chamorro", Code: "cha"}, //
{Name: "Cherokee", Code: "chr"}, //
{Name: "Chichewa", Code: "ny"},
{Name: "Chinese (Simplified)", Code: "zh"}, // "zh-cn" on Google
{Name: "Chinese (Traditional)", Code: "cht"}, // "zh-tw" on Google
{Name: "Chuvash", Code: "cv"},
{Name: "Coptic", Code: "cop"}, //
{Name: "Corsican", Code: "co"},
{Name: "Croatian", Code: "hr"},
{Name: "Czech", Code: "cs"},
{Name: "Danish", Code: "da"},
{Name: "Dhivehi", Code: "dv"}, //
{Name: "Dinka", Code: "dik"}, //
{Name: "Dutch", Code: "nl"},
{Name: "Dzongkha", Code: "dz"}, //
{Name: "English", Code: "en"},
{Name: "Esperanto", Code: "eo"},
{Name: "Estonian", Code: "et"},
{Name: "Ewe", Code: "ee"}, //
{Name: "Faroese", Code: "fo"}, //
{Name: "Fijian", Code: "fj"}, //
{Name: "Filipino", Code: "fil"}, // "tl" on Google
{Name: "Finnish", Code: "fi"},
{Name: "French", Code: "fr"},
{Name: "Frisian", Code: "fy"},
{Name: "Galela", Code: "gbi"}, //
{Name: "Galician", Code: "gl"},
{Name: "Ganda", Code: "lg"}, //
{Name: "Georgian", Code: "jy"}, // "ka" on Google
{Name: "German", Code: "de"},
{Name: "Greek", Code: "el"},
{Name: "Guerrero Amuzgo", Code: "amu"}, //
{Name: "Gujarati", Code: "gu"},
{Name: "Haitian Creole", Code: "ht"},
{Name: "Hausa", Code: "ha"},
{Name: "Hawaiian", Code: "haw"},
{Name: "Hebrew", Code: "he"}, // "iw" on Google
{Name: "Hindi", Code: "hi"},
{Name: "Hmong Daw", Code: "mww"}, //
{Name: "Hmong", Code: "hmn"}, // not in iciba
{Name: "Hungarian", Code: "hu"},
{Name: "Icelandic", Code: "is"},
{Name: "Igbo", Code: "ig"},
{Name: "Indonesian", Code: "id"},
{Name: "Irish", Code: "ga"},
{Name: "Italian", Code: "it"},
{Name: "Jacalteco", Code: "jac"}, //
{Name: "Japanese", Code: "ja"},
{Name: "Javanese", Code: "jv"}, // "jw" on Google
{Name: "Kabyle", Code: "kab"}, //
{Name: "Kannada", Code: "kn"},
{Name: "Kaqchikel", Code: "cak"}, //
{Name: "Kazakh", Code: "ka"}, // Google only has "kk"
{Name: "Kazakh (Cyrillic)", Code: "kk"}, // Google has it as just "Kazakh"
{Name: "Kekchí", Code: "kek"}, //
{Name: "Khmer", Code: "km"},
{Name: "Kinyarwanda", Code: "rw"},
{Name: "Kongo", Code: "kg"}, //
{Name: "Korean", Code: "ko"},
{Name: "Kurdish (Kurmanji)", Code: "ku"},
{Name: "Kyrgyz", Code: "ky"},
{Name: "Lao", Code: "lo"},
{Name: "Latin", Code: "la"},
{Name: "Latvian", Code: "lv"},
{Name: "Lingala", Code: "ln"}, //
{Name: "Lithuanian", Code: "lt"},
{Name: "Lukpa", Code: "dop"}, //
{Name: "Luxembourgish", Code: "lb"},
{Name: "Macedonian", Code: "mk"},
{Name: "Malagasy", Code: "mg"},
{Name: "Malay", Code: "ms"},
{Name: "Malayalam", Code: "ml"},
{Name: "Maltese", Code: "mt"},
{Name: "Mam", Code: "mam"}, //
{Name: "Manx", Code: "gv"}, //
{Name: "Maori", Code: "mi"},
{Name: "Marathi", Code: "mr"},
{Name: "Mari (Eastern)", Code: "mhr"}, //
{Name: "Mari (Western)", Code: "mrj"}, //
{Name: "Mongolian", Code: "mn"},
{Name: "Montenegrin", Code: "me"}, //
{Name: "Myanmar (Burmese)", Code: "my"},
{Name: "Nahuatl", Code: "nhg"}, //
{Name: "Ndyuka", Code: "djk"}, //
{Name: "Nepali", Code: "ne"},
{Name: "Norwegian", Code: "no"},
{Name: "Odia (Oriya)", Code: "or"},
{Name: "Ojibwa", Code: "ojb"},
{Name: "Oromo", Code: "om"}, //
{Name: "Ossetian", Code: "os"}, //
{Name: "Paite", Code: "pck"}, //
{Name: "Papiamento", Code: "pap"}, //
{Name: "Pashto", Code: "ps"},
{Name: "Persian", Code: "fa"},
{Name: "Polish", Code: "pl"},
{Name: "Portuguese", Code: "pt"},
{Name: "Potawatomi", Code: "pot"}, //
{Name: "Punjabi", Code: "pa"},
{Name: "Querétaro Otomi", Code: "otq"}, //
{Name: "Quiché", Code: "quc"}, //
{Name: "Quichua", Code: "quw"}, //
{Name: "Quiotepec Chinantec", Code: "chq"}, //
{Name: "Romani", Code: "rmn"}, //
{Name: "Romanian", Code: "ro"},
{Name: "Rundi", Code: "rn"}, //
{Name: "Russian", Code: "ru"},
{Name: "Samoan", Code: "sm"},
{Name: "Sango", Code: "sg"}, //
{Name: "Scots Gaelic", Code: "gd"},
{Name: "Serbian", Code: "sr"},
{Name: "Seselwa Creole French", Code: "crs"}, //
{Name: "Sesotho", Code: "st"},
{Name: "Shona", Code: "sn"},
{Name: "Shuar", Code: "jiv"}, //
{Name: "Sindhi", Code: "sd"},
{Name: "Sinhala", Code: "si"},
{Name: "Slovak", Code: "sk"},
{Name: "Slovenian", Code: "sl"},
{Name: "Somali", Code: "so"},
{Name: "Spanish", Code: "es"},
{Name: "Sundanese", Code: "su"},
{Name: "Swahili", Code: "sw"},
{Name: "Swedish", Code: "sv"},
{Name: "Syriac", Code: "syc"}, // considered "extinct" but is somehow supported
{Name: "Tachelhit", Code: "shi"}, //
{Name: "Tahitian", Code: "ty"}, //
{Name: "Tajik", Code: "tg"},
{Name: "Tamil", Code: "ta"},
{Name: "Tatar", Code: "tt"},
{Name: "Telugu", Code: "te"},
{Name: "Tetum", Code: "tet"}, //
{Name: "Thai", Code: "th"},
{Name: "Tigre", Code: "ti"}, //
{Name: "Tiwi", Code: "tw"}, //
{Name: "Tok Pisin", Code: "tpi"}, //
{Name: "Tonga", Code: "to"}, //
{Name: "Tsonga", Code: "ts"},
{Name: "Tswana", Code: "tn"}, //
{Name: "Turkish", Code: "tr"},
{Name: "Turkmen", Code: "tk"},
{Name: "Udmurt", Code: "udm"}, //
{Name: "Ukrainian", Code: "uk"},
{Name: "Uma", Code: "ppk"}, //
{Name: "Urdu", Code: "ur"},
{Name: "Uspanteco", Code: "usp"}, //
{Name: "Uyghur", Code: "uy"}, // "ug" on Google
{Name: "Uzbek", Code: "uz"},
{Name: "Venda", Code: "ve"}, //
{Name: "Vietnamese", Code: "vi"},
{Name: "Waray", Code: "war"}, //
{Name: "Welsh", Code: "cy"},
{Name: "Wolaitta", Code: "wal"}, //
{Name: "Wolof", Code: "wol"},
{Name: "Xhosa", Code: "xh"},
{Name: "Yiddish", Code: "yi"},
{Name: "Yoruba", Code: "yo"},
{Name: "Yucatán Maya", Code: "yua"}, //
{Name: "Zarma", Code: "dje"}, //
{Name: "Zulu", Code: "zu"},
}
func (_ *ICIBAEngine) SourceLanguages() ([]Language, error) { return icibaLanguages, nil }
func (_ *ICIBAEngine) TargetLanguages() ([]Language, error) { return icibaLanguages, nil }
func (_ *ICIBAEngine) SupportsAutodetect() bool { return true }
func (_ *ICIBAEngine) DetectLanguage(text string) (Language, error) { return Language{}, nil }
type icibaTranslateResponse struct {
Content struct {
From string `json:"from"`
Out string `json:"out"`
} `json:"content"`
}
func (_ *ICIBAEngine) Translate(text string, from Language, to Language) (TranslationResult, error) {
requestURL, err := url.Parse("https://ifanyi.iciba.com/index.php")
if err != nil {
// The URL is constant, so it should never fail.
panic(err)
}
query := url.Values{}
query.Add("c", "trans")
query.Add("m", "fy")
query.Add("client", "6")
query.Add("auth_user", "key_web_fanyi")
sum := md5.Sum([]byte(("6key_web_fanyiifanyiweb8hc9s98e" + text)))
query.Add("sign", hex.EncodeToString(sum[:])[:16])
requestURL.RawQuery = query.Encode()
formData := url.Values{}
formData.Add("from", from.Code)
formData.Add("to", to.Code)
formData.Add("q", text)
response, err := http.PostForm(requestURL.String(), formData)
if err != nil {
return TranslationResult{}, err
}
defer response.Body.Close()
if response.StatusCode != 200 {
return TranslationResult{}, fmt.Errorf("got status code %d from iCIBA", response.StatusCode)
}
var responseJSON icibaTranslateResponse
if err := json.NewDecoder(response.Body).Decode(&responseJSON); err != nil {
return TranslationResult{}, err
}
var sourceLanguage Language
for _, lang := range icibaLanguages {
if lang.Code == responseJSON.Content.From {
sourceLanguage = lang
break
}
}
if sourceLanguage == (Language{}) {
return TranslationResult{SourceLanguage: from, TranslatedText: responseJSON.Content.Out},
fmt.Errorf("language code \"%s\" is not in iCIBA's language list", responseJSON.Content.From)
}
return TranslationResult{
SourceLanguage: sourceLanguage,
TranslatedText: responseJSON.Content.Out,
}, nil
}

5
engines/language.go Normal file
View File

@@ -0,0 +1,5 @@
package engines
type Language struct {
Name, Code string
}

169
engines/libretranslate.go Normal file
View File

@@ -0,0 +1,169 @@
package engines
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
)
// LibreTranslateEngine is an engine that interfaces with any
// [LibreTranslate](https://github.com/LibreTranslate/LibreTranslate) instance.
type LibreTranslateEngine struct {
// InstanceURL is the URL to a LibreTranslate instance, for instance
// "https://libretranslate.com".
InstanceURL string
// APIKey is the API key for the given instance. If empty, then no API
// key will be sent along with requests to the instance.
//
// Some instances issue API keys to users so that they can have a
// higher rate limit. See
// https://github.com/LibreTranslate/LibreTranslate#manage-api-keys for
// more information.
APIKey string
}
func (_ *LibreTranslateEngine) InternalName() string { return "libre" }
func (_ *LibreTranslateEngine) DisplayName() string { return "LibreTranslate" }
type libreLanguagesResponse []struct {
Name string `json:"name"`
Code string `json:"code"`
}
func (e *LibreTranslateEngine) getLangs() ([]Language, error) {
response, err := http.Get(e.InstanceURL + "/languages")
if err != nil {
return nil, err
}
defer response.Body.Close()
if response.StatusCode != 200 {
return nil, fmt.Errorf("got status code %d from LibreTranslate API", response.StatusCode)
}
var langsResponse libreLanguagesResponse
if err := json.NewDecoder(response.Body).Decode(&langsResponse); err != nil {
return nil, err
}
langs := make([]Language, len(langsResponse))
for i, lang := range langsResponse {
langs[i] = Language{Name: lang.Name, Code: lang.Code}
}
return langs, nil
}
func (e *LibreTranslateEngine) SourceLanguages() ([]Language, error) { return e.getLangs() }
func (e *LibreTranslateEngine) TargetLanguages() ([]Language, error) { return e.getLangs() }
func (_ *LibreTranslateEngine) SupportsAutodetect() bool { return true }
type libreDetectResponse []struct {
Confidence float64 `json:"confidence"`
LanguageCode string `json:"language"`
}
func (e *LibreTranslateEngine) DetectLanguage(text string) (Language, error) {
formData := map[string]string{"q": text}
if e.APIKey != "" {
formData["api_key"] = e.APIKey
}
formDataJSON, err := json.Marshal(formData)
if err != nil {
return Language{}, err
}
response, err := http.Post(e.InstanceURL+"/detect", "application/json", bytes.NewBuffer(formDataJSON))
if err != nil {
return Language{}, err
}
defer response.Body.Close()
if response.StatusCode != 200 {
return Language{}, fmt.Errorf("got status code %d from LibreTranslate API", response.StatusCode)
}
var langs libreDetectResponse
if err := json.NewDecoder(response.Body).Decode(&langs); err != nil {
return Language{}, err
}
maxConfidenceLang := langs[0]
for _, lang := range langs[1:] {
if lang.Confidence > maxConfidenceLang.Confidence {
maxConfidenceLang = lang
}
}
engineLangs, err := e.getLangs()
if err != nil {
return Language{}, err
}
for _, lang := range engineLangs {
if lang.Code == maxConfidenceLang.LanguageCode {
return lang, nil
}
}
return Language{}, fmt.Errorf("language code \"%s\" is not in the instance's language list", maxConfidenceLang.LanguageCode)
}
type libreTranslateResponse struct {
TranslatedText string `json:"translatedText"`
}
func (e *LibreTranslateEngine) Translate(text string, from Language, to Language) (TranslationResult, error) {
formData := map[string]string{
"q": text,
"source": from.Code,
"target": to.Code,
}
if e.APIKey != "" {
formData["api_key"] = e.APIKey
}
formDataJSON, err := json.Marshal(formData)
if err != nil {
return TranslationResult{}, err
}
response, err := http.Post(e.InstanceURL+"/translate", "application/json", bytes.NewBuffer(formDataJSON))
if err != nil {
return TranslationResult{}, err
}
defer response.Body.Close()
if response.StatusCode != 200 {
return TranslationResult{}, fmt.Errorf("got status code %d from LibreTranslate API", response.StatusCode)
}
var responseJSON libreTranslateResponse
if err := json.NewDecoder(response.Body).Decode(&responseJSON); err != nil {
return TranslationResult{}, err
}
return TranslationResult{SourceLanguage: from, TranslatedText: responseJSON.TranslatedText}, nil
}