diff --git a/ci/translate/issue.tpl b/ci/translate/issue.tpl new file mode 100644 index 0000000..a5814b1 --- /dev/null +++ b/ci/translate/issue.tpl @@ -0,0 +1,20 @@ +--- +title: Missing Translations +--- +> As a developer I want my application to have correct translations in all available languages and not to have them to experience mixed translations in their native language and English. + +In order to achieve this we need to fix the following missing translations: + +{{ range $lang, $translation := .Translations -}} +{{ if MissingTranslations $lang -}} +### Language: `{{ $lang }}` + +Please add the following translations: +{{ range MissingTranslations $lang }} +- `{{ . }}` + > {{ English . }} +{{ end }} +_{{ Ping $lang }}_ + +{{ end -}} +{{ end -}} diff --git a/ci/translate/issuegen.go b/ci/translate/issuegen.go new file mode 100644 index 0000000..51b4931 --- /dev/null +++ b/ci/translate/issuegen.go @@ -0,0 +1,64 @@ +package main + +import ( + _ "embed" + "os" + "sort" + "strings" + "text/template" + + "github.com/pkg/errors" +) + +//go:embed issue.tpl +var issueTemplate string + +func generateIssue(tf translationFile) error { + fm := template.FuncMap{ + "English": tplEnglish(tf), + "MissingTranslations": tplMissingTranslations(tf), + "Ping": tplPing(tf), + } + + tpl, err := template.New("issue").Funcs(fm).Parse(issueTemplate) + if err != nil { + return errors.Wrap(err, "parsing issue template") + } + + f, err := os.Create(cfg.IssueFile) + if err != nil { + return errors.Wrap(err, "opening issue file") + } + defer f.Close() //nolint:errcheck // Short-lived fd-leak + + return errors.Wrap(tpl.Execute(f, tf), "executing issue template") +} + +func tplEnglish(tf translationFile) func(string) any { + return func(key string) any { + return tf.Reference.Translations[key] + } +} + +func tplMissingTranslations(tf translationFile) func(string) []string { + return func(lang string) []string { + missing, _, _ := tf.Translations[lang].Translations.GetErrorKeys(tf.Reference.Translations) + sort.Strings(missing) + return missing + } +} + +func tplPing(tf translationFile) func(string) string { + return func(lang string) string { + if len(tf.Translations[lang].Translators) == 0 { + return "No translators to ping for this language." + } + + var pings []string + for _, t := range tf.Translations[lang].Translators { + pings = append(pings, "@"+t) + } + + return strings.Join([]string{"Ping", strings.Join(pings, ", ")}, " ") + } +} diff --git a/ci/translate/translate.go b/ci/translate/main.go similarity index 88% rename from ci/translate/translate.go rename to ci/translate/main.go index 9cfb2b8..0e3d4ff 100644 --- a/ci/translate/translate.go +++ b/ci/translate/main.go @@ -20,31 +20,19 @@ import ( const deeplRequestTimeout = 10 * time.Second -type ( - translation map[string]any - translationFile struct { - Reference translationMapping `yaml:"reference"` - Translations map[string]*translationMapping `yaml:"translations"` - } - translationMapping struct { - DeeplLanguage string `yaml:"deeplLanguage,omitempty"` - LanguageKey string `yaml:"languageKey,omitempty"` - Translations translation `yaml:"translations"` - FormalTranslations translation `yaml:"formalTranslations,omitempty"` - } -) - var ( cfg = struct { AutoTranslate bool `flag:"auto-translate" default:"false" description:"Enable auto-translation through DeepL"` DeeplAPIEndpoint string `flag:"deepl-api-endpoint" default:"https://api-free.deepl.com/v2/translate" description:"DeepL API endpoint to request translations from"` DeeplAPIKey string `flag:"deepl-api-key" default:"" description:"API key for the DeepL API"` + IssueFile string `flag:"issue-file" default:"../../translate-issue.md" description:"Where to create the translate issue"` OutputFile string `flag:"output-file,o" default:"../../src/langs/langs.js" description:"Where to put rendered translations"` Template string `flag:"template" default:"../../src/langs/langs.tpl.js" description:"Template to load for translation JS file"` TranslationFile string `flag:"translation-file,t" default:"../../i18n.yaml" description:"File to use for translations"` LogLevel string `flag:"log-level" default:"info" description:"Log level (debug, info, warn, error, fatal)"` Verify bool `flag:"verify" default:"true" description:"Run verification against translation file"` VersionAndExit bool `flag:"version" default:"false" description:"Prints current version and exits"` + WriteIssueFile bool `flag:"write-issue-file" default:"false" description:"Generates an issue body for missing translations"` }{} version = "dev" @@ -113,6 +101,13 @@ func main() { if err = renderJSFile(tf); err != nil { logrus.WithError(err).Fatal("rendering JS output") } + + if cfg.WriteIssueFile { + logrus.Info("writing issue template...") + if err = generateIssue(tf); err != nil { + logrus.WithError(err).Fatal("generating issue template") + } + } } func autoTranslate(tf *translationFile) error { @@ -212,7 +207,11 @@ func fetchTranslation(srcLang, destLang, text string) (string, error) { if err != nil { return "", errors.Wrap(err, "executing request") } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + logrus.WithError(err).Error("closing response body (leaked fd)") + } + }() var payload struct { Translations []struct { @@ -237,7 +236,7 @@ func loadTranslationFile() (translationFile, error) { if err != nil { return tf, errors.Wrap(err, "opening translation file") } - defer f.Close() + defer f.Close() //nolint:errcheck // Short-lived fd-leak decoder := yaml.NewDecoder(f) decoder.KnownFields(true) @@ -262,11 +261,11 @@ func renderJSFile(tf translationFile) error { } if err = tpl.Execute(f, tf); err != nil { - f.Close() + f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak return errors.Wrap(err, "rendering js template") } - f.Close() + f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak return errors.Wrap(os.Rename(cfg.OutputFile+".tmp", cfg.OutputFile), "moving file in place") } @@ -277,18 +276,13 @@ func saveTranslationFile(tf translationFile) error { } encoder := yaml.NewEncoder(f) - encoder.SetIndent(2) + encoder.SetIndent(2) //nolint:gomnd if err = encoder.Encode(tf); err != nil { - f.Close() + f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak return errors.Wrap(err, "encoding translation file") } - f.Close() + f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak return errors.Wrap(os.Rename(cfg.TranslationFile+".tmp", cfg.TranslationFile), "moving file in place") } - -func (t translation) ToJSON() (string, error) { - j, err := json.Marshal(t) - return strings.ReplaceAll(string(j), "'", "\\'"), errors.Wrap(err, "marshalling JSON") -} diff --git a/ci/translate/translation.go b/ci/translate/translation.go new file mode 100644 index 0000000..de53c4e --- /dev/null +++ b/ci/translate/translation.go @@ -0,0 +1,66 @@ +package main + +import ( + "encoding/json" + "reflect" + "strings" + + "github.com/Luzifer/go_helpers/v2/str" + "github.com/pkg/errors" +) + +type ( + translation map[string]any + translationFile struct { + Reference translationMapping `yaml:"reference"` + Translations map[string]*translationMapping `yaml:"translations"` + } + translationMapping struct { + DeeplLanguage string `yaml:"deeplLanguage,omitempty"` + LanguageKey string `yaml:"languageKey,omitempty"` + Translators []string `yaml:"translators"` + Translations translation `yaml:"translations"` + FormalTranslations translation `yaml:"formalTranslations,omitempty"` + } +) + +func (t translation) ToJSON() (string, error) { + j, err := json.Marshal(t) + return strings.ReplaceAll(string(j), "'", "\\'"), errors.Wrap(err, "marshalling JSON") +} + +func (t translation) GetErrorKeys(ref translation) (missing, extra, wrongType []string) { + var ( + keys []string + keyType = map[string]reflect.Type{} + seenKeys []string + ) + + for k, v := range ref { + keys = append(keys, k) + keyType[k] = reflect.TypeOf(v) + } + + for k, v := range t { + if !str.StringInSlice(k, keys) { + // Contains extra key, is error + extra = append(extra, k) + continue // No further checks for that key + } + + seenKeys = append(seenKeys, k) + if kt := reflect.TypeOf(v); keyType[k] != kt { + // Type mismatches (i.e. string vs []string) + wrongType = append(wrongType, k) + continue + } + } + + for _, k := range keys { + if !str.StringInSlice(k, seenKeys) { + missing = append(missing, k) + } + } + + return missing, extra, wrongType +} diff --git a/ci/translate/verify.go b/ci/translate/verify.go index a4fd6d7..80e2337 100644 --- a/ci/translate/verify.go +++ b/ci/translate/verify.go @@ -1,41 +1,35 @@ package main import ( - "reflect" "regexp" + "sort" "github.com/pkg/errors" "github.com/sirupsen/logrus" - - "github.com/Luzifer/go_helpers/v2/str" ) var langKeyFormat = regexp.MustCompile(`^[a-z]{2}(-[A-Z]{2})?$`) func verify(tf translationFile) error { - var ( - err error - keys []string - keyType = map[string]reflect.Type{} - ) - - for k, v := range tf.Reference.Translations { - keys = append(keys, k) - keyType[k] = reflect.TypeOf(v) - } + var err error if !langKeyFormat.MatchString(tf.Reference.LanguageKey) { return errors.New("reference contains invalid languageKey") } - if len(keys) == 0 { + if len(tf.Reference.Translations) == 0 { return errors.New("reference does not contain translations") } - logrus.Infof("found %d translation keys in reference", len(keys)) + logrus.Infof("found %d translation keys in reference", len(tf.Reference.Translations)) if tf.Reference.FormalTranslations != nil { - if verifyTranslationKeys(logrus.NewEntry(logrus.StandardLogger()), tf.Reference.FormalTranslations, keys, keyType, false); err != nil { + if verifyTranslationKeys( + logrus.NewEntry(logrus.StandardLogger()), + tf.Reference.FormalTranslations, + tf.Reference.Translations, + false, + ); err != nil { return errors.New("reference contains error in formalTranslations") } } @@ -54,8 +48,18 @@ func verify(tf translationFile) error { logger.Info("no deeplLanguage is set") } - hadErrors = hadErrors || verifyTranslationKeys(logger, tm.Translations, keys, keyType, true) - hadErrors = hadErrors || verifyTranslationKeys(logger, tm.FormalTranslations, keys, keyType, false) + hadErrors = hadErrors || verifyTranslationKeys( + logger, + tm.Translations, + tf.Reference.Translations, + true, + ) + hadErrors = hadErrors || verifyTranslationKeys( + logger, + tm.FormalTranslations, + tf.Reference.Translations, + false, + ) } if hadErrors { @@ -64,31 +68,27 @@ func verify(tf translationFile) error { return nil } -func verifyTranslationKeys(logger *logrus.Entry, t translation, keys []string, keyType map[string]reflect.Type, warnMissing bool) (hadErrors bool) { - var seenKeys []string +//revive:disable-next-line:flag-parameter +func verifyTranslationKeys(logger *logrus.Entry, t, ref translation, warnMissing bool) (hadErrors bool) { + missing, extra, wrongType := t.GetErrorKeys(ref) - for k, v := range t { - keyLogger := logger.WithField("translation_key", k) - if !str.StringInSlice(k, keys) { - // Contains extra key, is error - hadErrors = true - keyLogger.Error("extra key found") - continue // No further checks for that key - } + sort.Strings(extra) + sort.Strings(missing) + sort.Strings(wrongType) - seenKeys = append(seenKeys, k) - if kt := reflect.TypeOf(v); keyType[k] != kt { - // Type mismatches (i.e. string vs []string) - hadErrors = true - keyLogger.Errorf("key has invalid type %s != %s", kt, keyType[k]) - } + for _, k := range extra { + logger.WithField("translation_key", k).Error("extra key found") } - for _, k := range keys { - if warnMissing && !str.StringInSlice(k, seenKeys) { + for _, k := range wrongType { + logger.WithField("translation_key", k).Error("key has invalid type") + } + + if warnMissing { + for _, k := range missing { logger.WithField("translation_key", k).Warn("missing translation") } } - return hadErrors + return len(extra)+len(wrongType) > 0 }