Add automated Issue generation for missing translations (#119)

This commit is contained in:
Knut Ahlers 2023-10-05 15:29:06 +02:00 committed by GitHub
parent 8540d4016c
commit 0f544d9ac7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 238 additions and 64 deletions

View File

@ -8,6 +8,7 @@ on:
permissions:
contents: write
issues: write
jobs:
test-and-build:
@ -70,6 +71,15 @@ jobs:
- name: Generate (and validate) translations
run: make translate
- name: Update Translations Issue
uses: JasonEtco/create-an-issue@v2
if: github.ref == 'refs/heads/master'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
filename: translate-issue.md
update_existing: true
- name: Build release
run: make publish
env:

1
.gitignore vendored
View File

@ -11,3 +11,4 @@ frontend/webfonts
frontend/*.woff2
node_modules
ots
translate-issue.md

View File

@ -26,7 +26,7 @@ publish: download_libs generate-inner generate-apidocs
bash ./ci/build.sh
translate:
cd ci/translate && go run .
cd ci/translate && go run . --write-issue-file
# -- Download / refresh external libraries --

22
ci/translate/issue.tpl.md Normal file
View File

@ -0,0 +1,22 @@
---
title: Missing Translations
---
> As a developer I want my application to have correct translations in all available languages and not to have them to experience mixed translations in their native language and English.
**In order to achieve this we need to fix the following missing translations.**
To do so please either **create a pull-request** updating the `i18n.yaml` in the root of the repository and add the missing translations to the corresponding language or **just leave a comment** below and ping @Luzifer in your comment. He then will integrate the new translation strings and mark your comment hidden after this issue has been automatically updated (kind of a to-do list for translations until we have something better in place).
{{ range $lang, $translation := .Translations -}}
{{ if MissingTranslations $lang -}}
### Language: `{{ $lang }}`
Please add the following translations:
{{ range MissingTranslations $lang }}
- `{{ . }}`
> {{ English . }}
{{ end }}
_{{ Ping $lang }}_
{{ end -}}
{{ end -}}

64
ci/translate/issuegen.go Normal file
View File

@ -0,0 +1,64 @@
package main
import (
_ "embed"
"os"
"sort"
"strings"
"text/template"
"github.com/pkg/errors"
)
//go:embed issue.tpl.md
var issueTemplate string
func generateIssue(tf translationFile) error {
fm := template.FuncMap{
"English": tplEnglish(tf),
"MissingTranslations": tplMissingTranslations(tf),
"Ping": tplPing(tf),
}
tpl, err := template.New("issue").Funcs(fm).Parse(issueTemplate)
if err != nil {
return errors.Wrap(err, "parsing issue template")
}
f, err := os.Create(cfg.IssueFile)
if err != nil {
return errors.Wrap(err, "opening issue file")
}
defer f.Close() //nolint:errcheck // Short-lived fd-leak
return errors.Wrap(tpl.Execute(f, tf), "executing issue template")
}
func tplEnglish(tf translationFile) func(string) any {
return func(key string) any {
return tf.Reference.Translations[key]
}
}
func tplMissingTranslations(tf translationFile) func(string) []string {
return func(lang string) []string {
missing, _, _ := tf.Translations[lang].Translations.GetErrorKeys(tf.Reference.Translations)
sort.Strings(missing)
return missing
}
}
func tplPing(tf translationFile) func(string) string {
return func(lang string) string {
if len(tf.Translations[lang].Translators) == 0 {
return "No translators to ping for this language."
}
var pings []string
for _, t := range tf.Translations[lang].Translators {
pings = append(pings, "@"+t)
}
return strings.Join([]string{"Ping", strings.Join(pings, ", ")}, " ")
}
}

View File

@ -20,31 +20,19 @@ import (
const deeplRequestTimeout = 10 * time.Second
type (
translation map[string]any
translationFile struct {
Reference translationMapping `yaml:"reference"`
Translations map[string]*translationMapping `yaml:"translations"`
}
translationMapping struct {
DeeplLanguage string `yaml:"deeplLanguage,omitempty"`
LanguageKey string `yaml:"languageKey,omitempty"`
Translations translation `yaml:"translations"`
FormalTranslations translation `yaml:"formalTranslations,omitempty"`
}
)
var (
cfg = struct {
AutoTranslate bool `flag:"auto-translate" default:"false" description:"Enable auto-translation through DeepL"`
DeeplAPIEndpoint string `flag:"deepl-api-endpoint" default:"https://api-free.deepl.com/v2/translate" description:"DeepL API endpoint to request translations from"`
DeeplAPIKey string `flag:"deepl-api-key" default:"" description:"API key for the DeepL API"`
IssueFile string `flag:"issue-file" default:"../../translate-issue.md" description:"Where to create the translate issue"`
OutputFile string `flag:"output-file,o" default:"../../src/langs/langs.js" description:"Where to put rendered translations"`
Template string `flag:"template" default:"../../src/langs/langs.tpl.js" description:"Template to load for translation JS file"`
TranslationFile string `flag:"translation-file,t" default:"../../i18n.yaml" description:"File to use for translations"`
LogLevel string `flag:"log-level" default:"info" description:"Log level (debug, info, warn, error, fatal)"`
Verify bool `flag:"verify" default:"true" description:"Run verification against translation file"`
VersionAndExit bool `flag:"version" default:"false" description:"Prints current version and exits"`
WriteIssueFile bool `flag:"write-issue-file" default:"false" description:"Generates an issue body for missing translations"`
}{}
version = "dev"
@ -113,6 +101,13 @@ func main() {
if err = renderJSFile(tf); err != nil {
logrus.WithError(err).Fatal("rendering JS output")
}
if cfg.WriteIssueFile {
logrus.Info("writing issue template...")
if err = generateIssue(tf); err != nil {
logrus.WithError(err).Fatal("generating issue template")
}
}
}
func autoTranslate(tf *translationFile) error {
@ -212,7 +207,11 @@ func fetchTranslation(srcLang, destLang, text string) (string, error) {
if err != nil {
return "", errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
defer func() {
if err := resp.Body.Close(); err != nil {
logrus.WithError(err).Error("closing response body (leaked fd)")
}
}()
var payload struct {
Translations []struct {
@ -237,7 +236,7 @@ func loadTranslationFile() (translationFile, error) {
if err != nil {
return tf, errors.Wrap(err, "opening translation file")
}
defer f.Close()
defer f.Close() //nolint:errcheck // Short-lived fd-leak
decoder := yaml.NewDecoder(f)
decoder.KnownFields(true)
@ -262,11 +261,11 @@ func renderJSFile(tf translationFile) error {
}
if err = tpl.Execute(f, tf); err != nil {
f.Close()
f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak
return errors.Wrap(err, "rendering js template")
}
f.Close()
f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak
return errors.Wrap(os.Rename(cfg.OutputFile+".tmp", cfg.OutputFile), "moving file in place")
}
@ -277,18 +276,13 @@ func saveTranslationFile(tf translationFile) error {
}
encoder := yaml.NewEncoder(f)
encoder.SetIndent(2)
encoder.SetIndent(2) //nolint:gomnd
if err = encoder.Encode(tf); err != nil {
f.Close()
f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak
return errors.Wrap(err, "encoding translation file")
}
f.Close()
f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak
return errors.Wrap(os.Rename(cfg.TranslationFile+".tmp", cfg.TranslationFile), "moving file in place")
}
func (t translation) ToJSON() (string, error) {
j, err := json.Marshal(t)
return strings.ReplaceAll(string(j), "'", "\\'"), errors.Wrap(err, "marshalling JSON")
}

View File

@ -0,0 +1,66 @@
package main
import (
"encoding/json"
"reflect"
"strings"
"github.com/Luzifer/go_helpers/v2/str"
"github.com/pkg/errors"
)
type (
translation map[string]any
translationFile struct {
Reference translationMapping `yaml:"reference"`
Translations map[string]*translationMapping `yaml:"translations"`
}
translationMapping struct {
DeeplLanguage string `yaml:"deeplLanguage,omitempty"`
LanguageKey string `yaml:"languageKey,omitempty"`
Translators []string `yaml:"translators"`
Translations translation `yaml:"translations"`
FormalTranslations translation `yaml:"formalTranslations,omitempty"`
}
)
func (t translation) ToJSON() (string, error) {
j, err := json.Marshal(t)
return strings.ReplaceAll(string(j), "'", "\\'"), errors.Wrap(err, "marshalling JSON")
}
func (t translation) GetErrorKeys(ref translation) (missing, extra, wrongType []string) {
var (
keys []string
keyType = map[string]reflect.Type{}
seenKeys []string
)
for k, v := range ref {
keys = append(keys, k)
keyType[k] = reflect.TypeOf(v)
}
for k, v := range t {
if !str.StringInSlice(k, keys) {
// Contains extra key, is error
extra = append(extra, k)
continue // No further checks for that key
}
seenKeys = append(seenKeys, k)
if kt := reflect.TypeOf(v); keyType[k] != kt {
// Type mismatches (i.e. string vs []string)
wrongType = append(wrongType, k)
continue
}
}
for _, k := range keys {
if !str.StringInSlice(k, seenKeys) {
missing = append(missing, k)
}
}
return missing, extra, wrongType
}

View File

@ -1,41 +1,35 @@
package main
import (
"reflect"
"regexp"
"sort"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/Luzifer/go_helpers/v2/str"
)
var langKeyFormat = regexp.MustCompile(`^[a-z]{2}(-[A-Z]{2})?$`)
func verify(tf translationFile) error {
var (
err error
keys []string
keyType = map[string]reflect.Type{}
)
for k, v := range tf.Reference.Translations {
keys = append(keys, k)
keyType[k] = reflect.TypeOf(v)
}
var err error
if !langKeyFormat.MatchString(tf.Reference.LanguageKey) {
return errors.New("reference contains invalid languageKey")
}
if len(keys) == 0 {
if len(tf.Reference.Translations) == 0 {
return errors.New("reference does not contain translations")
}
logrus.Infof("found %d translation keys in reference", len(keys))
logrus.Infof("found %d translation keys in reference", len(tf.Reference.Translations))
if tf.Reference.FormalTranslations != nil {
if verifyTranslationKeys(logrus.NewEntry(logrus.StandardLogger()), tf.Reference.FormalTranslations, keys, keyType, false); err != nil {
if verifyTranslationKeys(
logrus.NewEntry(logrus.StandardLogger()),
tf.Reference.FormalTranslations,
tf.Reference.Translations,
false,
); err != nil {
return errors.New("reference contains error in formalTranslations")
}
}
@ -54,8 +48,18 @@ func verify(tf translationFile) error {
logger.Info("no deeplLanguage is set")
}
hadErrors = hadErrors || verifyTranslationKeys(logger, tm.Translations, keys, keyType, true)
hadErrors = hadErrors || verifyTranslationKeys(logger, tm.FormalTranslations, keys, keyType, false)
hadErrors = hadErrors || verifyTranslationKeys(
logger,
tm.Translations,
tf.Reference.Translations,
true,
)
hadErrors = hadErrors || verifyTranslationKeys(
logger,
tm.FormalTranslations,
tf.Reference.Translations,
false,
)
}
if hadErrors {
@ -64,31 +68,27 @@ func verify(tf translationFile) error {
return nil
}
func verifyTranslationKeys(logger *logrus.Entry, t translation, keys []string, keyType map[string]reflect.Type, warnMissing bool) (hadErrors bool) {
var seenKeys []string
//revive:disable-next-line:flag-parameter
func verifyTranslationKeys(logger *logrus.Entry, t, ref translation, warnMissing bool) (hadErrors bool) {
missing, extra, wrongType := t.GetErrorKeys(ref)
for k, v := range t {
keyLogger := logger.WithField("translation_key", k)
if !str.StringInSlice(k, keys) {
// Contains extra key, is error
hadErrors = true
keyLogger.Error("extra key found")
continue // No further checks for that key
sort.Strings(extra)
sort.Strings(missing)
sort.Strings(wrongType)
for _, k := range extra {
logger.WithField("translation_key", k).Error("extra key found")
}
seenKeys = append(seenKeys, k)
if kt := reflect.TypeOf(v); keyType[k] != kt {
// Type mismatches (i.e. string vs []string)
hadErrors = true
keyLogger.Errorf("key has invalid type %s != %s", kt, keyType[k])
}
for _, k := range wrongType {
logger.WithField("translation_key", k).Error("key has invalid type")
}
for _, k := range keys {
if warnMissing && !str.StringInSlice(k, seenKeys) {
if warnMissing {
for _, k := range missing {
logger.WithField("translation_key", k).Warn("missing translation")
}
}
return hadErrors
return len(extra)+len(wrongType) > 0
}

View File

@ -1,6 +1,8 @@
reference:
deeplLanguage: en
languageKey: en
translators:
- Luzifer
translations:
alert-secret-not-found: This is not the secret you are looking for… - If you expected the secret to be here it might be compromised as someone else might have opened the link already.
alert-something-went-wrong: Something went wrong. I'm very sorry about this…
@ -43,6 +45,7 @@ reference:
title-secret-created: Secret created!
translations:
ca:
translators: []
translations:
alert-secret-not-found: Aquest no és el secret que busques… - Si esperaves que el secret estiguera ací, és possible que s'haja vist compromés, ja que una altra persona podria haver obert l'enllaç en comptes de tu.
alert-something-went-wrong: Alguna cosa ha eixit malament. Ens sap molt greu…
@ -79,6 +82,8 @@ translations:
title-secret-created: Secret creat!
de:
deeplLanguage: de
translators:
- Luzifer
translations:
alert-secret-not-found: Das ist nicht das Secret, was du suchst… - Falls du diesen Link noch nicht selbst geöffnet hast, könnte das Secret kompromittiert sein, da jemand anderes den Link geöffnet haben könnte.
alert-something-went-wrong: Irgendwas ging schief. Entschuldigung…
@ -138,6 +143,7 @@ translations:
title-new-secret: Ein neues Secret erstellen
es:
deeplLanguage: es
translators: []
translations:
alert-secret-not-found: Este no es el secreto que buscas… - Si esperabas que el secreto estuviera aquí, es posible que se haya visto comprometido, ya que otra persona podría haber abierto el enlace en tu lugar.
alert-something-went-wrong: Algo ha salido mal. Lo sentimos mucho…
@ -174,6 +180,7 @@ translations:
title-secret-created: ¡Secreto creado!
fr:
deeplLanguage: fr
translators: []
translations:
alert-secret-not-found: Ce secret n'est pas celui que vous cherchez… - Si vous comptiez trouvez ce secret ici, il a pu être compromis car quelqu'un a probablement déjà ouvert le lien.
alert-something-went-wrong: Un problème est survenu. Nous en sommes désolés…
@ -207,6 +214,7 @@ translations:
title-secret-created: Secret créé!
lv:
deeplLanguage: lv
translators: []
translations:
alert-secret-not-found: <strong>Ziņa nav atrasta!</strong>&hellip; - Ja ievadītā saite ir pareiza, tad ir beidzies ziņas glabāšanas laiks, vai arī tā jau vienreiz ir atvērta.
alert-something-went-wrong: Neparedzēta sistēmas kļūda. Atvainojiet par sagādātajām neērtībām&hellip;
@ -240,6 +248,7 @@ translations:
title-secret-created: Ziņa nošifrēta!
nl:
deeplLanguage: nl
translators: []
translations:
alert-secret-not-found: De gegevens die je zocht bestaan niet (meer)&hellip; - Als je hier informatie verwachtte dan is de link mogelijk al door iemand anders bekeken!
alert-something-went-wrong: Er ging iets verkeerd, sorry&hellip;
@ -273,6 +282,7 @@ translations:
title-secret-created: Vertrouwelijke info opgeslaan!
pl:
deeplLanguage: pl
translators: []
translations:
alert-secret-not-found: TO nie jest sekret, którego szukasz&hellip; - Jeśli spodziewałeś się tu sekretu, to może być on zagrożony, ponieważ ktoś inny mógł już otworzyć ten link.
alert-something-went-wrong: Coś poszło nie tak. Bardzo mi przykro&hellip;
@ -309,6 +319,7 @@ translations:
title-secret-created: Sekret utworzony!
pt-BR:
deeplLanguage: pt-BR
translators: []
translations:
alert-secret-not-found: Esta não é o segredo que você está procurando… - Se você esperava que o segredo estaria aqui, ele pode ter sido comprometido por alguém que já acessou o link.
alert-something-went-wrong: Desculpe, algo deu errado…
@ -342,6 +353,7 @@ translations:
title-secret-created: Segredo criado!
ru:
deeplLanguage: ru
translators: []
translations:
alert-secret-not-found: Секрет недоступен&hellip; - Помните, он может быть скомпрометирован. Возможно кто-то другой уже открыл вашу ссылку.
alert-something-went-wrong: Что-то пошло не так. Приносим свои извинения&hellip;
@ -375,6 +387,7 @@ translations:
title-secret-created: Секрет создан!
sv:
deeplLanguage: sv
translators: []
translations:
alert-secret-not-found: Hemlighet hittades inte&hellip; - Om du förväntade dig att hemligheten skulle finnas här kan den vara röjd då någon annan kan ha öppnat denna länk tidigare.
alert-something-went-wrong: Något gick fel. Jag ber om ursäkt för detta!&hellip;
@ -408,6 +421,7 @@ translations:
title-secret-created: Hemlighet skapad!
tr:
deeplLanguage: tr
translators: []
translations:
alert-secret-not-found: Aradığınız sır bu değil… - Sırrın burada olmasını bekliyorsanız, bu link başkası tarafından açılmış ve sırrınız tehlikede olabilir.
alert-something-went-wrong: Bir şeyler ters gitti. Bunun için çok üzgünüm…
@ -441,6 +455,7 @@ translations:
title-secret-created: Sır oluşturuldu!
uk:
deeplLanguage: uk
translators: []
translations:
alert-secret-not-found: Це не секрет, який ви шукаєте&hellip; - Якщо ви очікували, що секрет буде тут, він міг бути скомпрометований, оскільки хтось інший міг уже відкрити посилання.
alert-something-went-wrong: Щось пішло не так. Ми дуже шкодуємо про це&hellip;
@ -476,6 +491,7 @@ translations:
title-secret-create-disabled: Створення секрету вимкнено…
title-secret-created: Секрет створений!
zh:
translators: []
translations:
alert-secret-not-found: 该秘密不存在 - 如果这与您的预期不符,那么该链接可能已经泄露,且您的秘密已经被其他人查看了。
alert-something-went-wrong: 运行异常,对此我深感抱歉…
@ -511,6 +527,7 @@ translations:
title-secret-create-disabled: 创建秘密被禁止…
title-secret-created: 秘密已创建!
zh-TW:
translators: []
translations:
alert-secret-not-found: 這不是您正在尋找的機密&hellip; - 如果您期望機密會出現在這裡,它可能已經被泄漏了,因為可能有其他人已經打開了此連結。
alert-something-went-wrong: 看樣子出了一些問題,對此我非常抱歉&hellip;