diff --git a/dev.md b/dev.md new file mode 100644 index 00000000..a55181eb --- /dev/null +++ b/dev.md @@ -0,0 +1,8 @@ +# Development notes + +## How to upgrade CLDR data + +1. Go to http://cldr.unicode.org/index/downloads to find the latest version. +1. Download the latest version of cldr-common (e.g. http://unicode.org/Public/cldr/33/cldr-common-33.0.zip) +1. Unzip and copy `common/supplemental/plurals.xml` to `v2/i18n/internal/plural/codegen/plurals.xml` +1. Run `generate.sh` in `v2/i18n/internal/plural/codegen/` diff --git a/v2/goi18n/merge_command.go b/v2/goi18n/merge_command.go index 32e3d1d3..bf13e475 100644 --- a/v2/goi18n/merge_command.go +++ b/v2/goi18n/merge_command.go @@ -148,8 +148,7 @@ func merge(messageFiles map[string][]byte, sourceLanguageTag language.Tag, outdi if dstLangTag == sourceLanguageTag { continue } - dstBase, _ := dstLangTag.Base() - pluralRule := pluralRules[dstBase] + pluralRule := pluralRules.Rule(dstLangTag) if pluralRule == nil { // Non-standard languages not supported because // we don't know if translations are complete or not. @@ -202,8 +201,7 @@ func merge(messageFiles map[string][]byte, sourceLanguageTag language.Tag, outdi active[langTag] = messageTemplates continue } - base, _ := langTag.Base() - pluralRule := pluralRules[base] + pluralRule := pluralRules.Rule(langTag) if pluralRule == nil { // Non-standard languages not supported because // we don't know if translations are complete or not. diff --git a/v2/i18n/bundle.go b/v2/i18n/bundle.go index 83ffd05d..4bb89e1c 100644 --- a/v2/i18n/bundle.go +++ b/v2/i18n/bundle.go @@ -19,7 +19,7 @@ type UnmarshalFunc = internal.UnmarshalFunc // that is initialized early in the application's lifecycle. type Bundle struct { messageTemplates map[language.Tag]map[string]*internal.MessageTemplate - pluralRules map[language.Base]*plural.Rule + pluralRules plural.Rules unmarshalFuncs map[string]UnmarshalFunc defaultTag language.Tag tags []language.Tag @@ -40,11 +40,6 @@ func NewBundle(defaultTag language.Tag) *Bundle { return b } -// RegisterPluralRule registers a plural rule for a language base. -// func (b *Bundle) RegisterPluralRule(base language.Base, rule *plural.Rule) { -// b.pluralRules[base] = rule -// } - // RegisterUnmarshalFunc registers an UnmarshalFunc for format. func (b *Bundle) RegisterUnmarshalFunc(format string, unmarshalFunc UnmarshalFunc) { if b.unmarshalFuncs == nil { @@ -104,12 +99,10 @@ func (b *Bundle) AddMessages(tag language.Tag, messages ...*Message) error { if b.pluralRules == nil { b.pluralRules = plural.DefaultRules() } - base, _ := tag.Base() - pluralRule := b.pluralRules[base] + pluralRule := b.pluralRules.Rule(tag) if pluralRule == nil { - return fmt.Errorf("no plural rule registered for %s", base) + return fmt.Errorf("no plural rule registered for %s", tag) } - b.pluralRules[base] = pluralRule if b.messageTemplates == nil { b.messageTemplates = map[language.Tag]map[string]*internal.MessageTemplate{} } diff --git a/v2/i18n/localizer.go b/v2/i18n/localizer.go index bf524138..41dc9503 100644 --- a/v2/i18n/localizer.go +++ b/v2/i18n/localizer.go @@ -78,11 +78,11 @@ func (e *messageNotFoundErr) Error() string { type pluralizeErr struct { messageID string - base language.Base + tag language.Tag } func (e *pluralizeErr) Error() string { - return fmt.Sprintf("unable to pluralize %q because there no plural rule for %q", e.messageID, e.base) + return fmt.Sprintf("unable to pluralize %q because there no plural rule for %q", e.messageID, e.tag) } // Localize returns a localized message. @@ -110,10 +110,9 @@ func (l *Localizer) Localize(lc *LocalizeConfig) (string, error) { if template == nil { return "", &messageNotFoundErr{messageID: messageID} } - base, _ := tag.Base() - pluralForm := l.pluralForm(base, operands) + pluralForm := l.pluralForm(tag, operands) if pluralForm == plural.Invalid { - return "", &pluralizeErr{messageID: messageID, base: base} + return "", &pluralizeErr{messageID: messageID, tag: tag} } return template.Execute(pluralForm, templateData) } @@ -172,11 +171,11 @@ func (l *Localizer) matchTemplate(id string, matcher language.Matcher, tags []la return tag, nil } -func (l *Localizer) pluralForm(base language.Base, operands *plural.Operands) plural.Form { +func (l *Localizer) pluralForm(tag language.Tag, operands *plural.Operands) plural.Form { if operands == nil { return plural.Other } - pluralRule := l.bundle.pluralRules[base] + pluralRule := l.bundle.pluralRules.Rule(tag) if pluralRule == nil { return plural.Invalid } diff --git a/v2/internal/plural/codegen/main.go b/v2/internal/plural/codegen/main.go index 14e5fc93..2b58c384 100644 --- a/v2/internal/plural/codegen/main.go +++ b/v2/internal/plural/codegen/main.go @@ -80,11 +80,9 @@ var codeTemplate = template.Must(template.New("rule").Parse(`// This file is gen package plural -import "golang.org/x/text/language" - // DefaultRules returns a map of Rules generated from CLDR language data. -func DefaultRules() map[language.Base]*Rule { - rules := make(map[language.Base]*Rule) +func DefaultRules() Rules { + rules := Rules{} {{range .PluralGroups}} addPluralRules(rules, {{printf "%#v" .SplitLocales}}, &Rule{ diff --git a/v2/internal/plural/codegen/plurals.xml b/v2/internal/plural/codegen/plurals.xml index 3310c8ee..2704a8e7 100644 --- a/v2/internal/plural/codegen/plurals.xml +++ b/v2/internal/plural/codegen/plurals.xml @@ -6,7 +6,7 @@ CLDR data files are interpreted according to the LDML specification (http://unic For terms of use, see http://www.unicode.org/copyright.html --> - + @@ -30,7 +30,7 @@ For terms of use, see http://www.unicode.org/copyright.html i = 0..1 @integer 0, 1 @decimal 0.0~1.5 @integer 2~17, 100, 1000, 10000, 100000, 1000000, … @decimal 2.0~3.5, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, … - + i = 1 and v = 0 @integer 1 @integer 0, 2~16, 100, 1000, 10000, 100000, 1000000, … @decimal 0.0~1.5, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, … @@ -46,7 +46,7 @@ For terms of use, see http://www.unicode.org/copyright.html n = 0..1 or n = 11..99 @integer 0, 1, 11~24 @decimal 0.0, 1.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0 @integer 2~10, 100~106, 1000, 10000, 100000, 1000000, … @decimal 0.1~0.9, 1.1~1.7, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, … - + n = 1 @integer 1 @decimal 1.0, 1.00, 1.000, 1.0000 @integer 0, 2~16, 100, 1000, 10000, 100000, 1000000, … @decimal 0.0~0.9, 1.1~1.6, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, … @@ -59,8 +59,8 @@ For terms of use, see http://www.unicode.org/copyright.html @integer 0, 2~16, 100, 1000, 10000, 100000, 1000000, … @decimal 0.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, … - v = 0 and i % 10 = 1 or f % 10 = 1 @integer 1, 11, 21, 31, 41, 51, 61, 71, 101, 1001, … @decimal 0.1, 1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1, 10.1, 100.1, 1000.1, … - @integer 0, 2~10, 12~17, 100, 1000, 10000, 100000, 1000000, … @decimal 0.0, 0.2~1.0, 1.2~1.7, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, … + v = 0 and i % 10 = 1 and i % 100 != 11 or f % 10 = 1 and f % 100 != 11 @integer 1, 21, 31, 41, 51, 61, 71, 81, 101, 1001, … @decimal 0.1, 1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1, 10.1, 100.1, 1000.1, … + @integer 0, 2~16, 100, 1000, 10000, 100000, 1000000, … @decimal 0.0, 0.2~1.0, 1.2~1.7, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, … v = 0 and i = 1,2,3 or v = 0 and i % 10 != 4,6,9 or v != 0 and f % 10 != 4,6,9 @integer 0~3, 5, 7, 8, 10~13, 15, 17, 18, 20, 21, 100, 1000, 10000, 100000, 1000000, … @decimal 0.0~0.3, 0.5, 0.7, 0.8, 1.0~1.3, 1.5, 1.7, 1.8, 2.0, 2.1, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, … diff --git a/v2/internal/plural/rule.go b/v2/internal/plural/rule.go index 41aa4c9d..0869c84f 100644 --- a/v2/internal/plural/rule.go +++ b/v2/internal/plural/rule.go @@ -1,6 +1,8 @@ package plural -import "golang.org/x/text/language" +import ( + "golang.org/x/text/language" +) // Rule defines the CLDR plural rules for a language. // http://www.unicode.org/cldr/charts/latest/supplemental/language_plural_rules.html @@ -10,13 +12,13 @@ type Rule struct { PluralFormFunc func(*Operands) Form } -func addPluralRules(rules map[language.Base]*Rule, ids []string, ps *Rule) { +func addPluralRules(rules Rules, ids []string, ps *Rule) { for _, id := range ids { if id == "root" { continue } - base := language.MustParseBase(id) - rules[base] = ps + tag := language.MustParse(id) + rules[tag] = ps } } diff --git a/v2/internal/plural/rule_gen.go b/v2/internal/plural/rule_gen.go index 95ff7ac0..7cc6fc7b 100644 --- a/v2/internal/plural/rule_gen.go +++ b/v2/internal/plural/rule_gen.go @@ -2,11 +2,9 @@ package plural -import "golang.org/x/text/language" - // DefaultRules returns a map of Rules generated from CLDR language data. -func DefaultRules() map[language.Base]*Rule { - rules := make(map[language.Base]*Rule) +func DefaultRules() Rules { + rules := Rules{} addPluralRules(rules, []string{"bm", "bo", "dz", "id", "ig", "ii", "in", "ja", "jbo", "jv", "jw", "kde", "kea", "km", "ko", "lkt", "lo", "ms", "my", "nqo", "root", "sah", "ses", "sg", "th", "to", "vi", "wo", "yo", "yue", "zh"}, &Rule{ PluralForms: newPluralFormSet(Other), @@ -45,7 +43,7 @@ func DefaultRules() map[language.Base]*Rule { return Other }, }) - addPluralRules(rules, []string{"ast", "ca", "de", "en", "et", "fi", "fy", "gl", "it", "ji", "nl", "sv", "sw", "ur", "yi"}, &Rule{ + addPluralRules(rules, []string{"ast", "ca", "de", "en", "et", "fi", "fy", "gl", "io", "it", "ji", "nl", "pt_PT", "scn", "sv", "sw", "ur", "yi"}, &Rule{ PluralForms: newPluralFormSet(One, Other), PluralFormFunc: func(ops *Operands) Form { // i = 1 and v = 0 @@ -87,7 +85,7 @@ func DefaultRules() map[language.Base]*Rule { return Other }, }) - addPluralRules(rules, []string{"af", "asa", "az", "bem", "bez", "bg", "brx", "ce", "cgg", "chr", "ckb", "dv", "ee", "el", "eo", "es", "eu", "fo", "fur", "gsw", "ha", "haw", "hu", "jgo", "jmc", "ka", "kaj", "kcg", "kk", "kkj", "kl", "ks", "ksb", "ku", "ky", "lb", "lg", "mas", "mgo", "ml", "mn", "nah", "nb", "nd", "ne", "nn", "nnh", "no", "nr", "ny", "nyn", "om", "or", "os", "pap", "ps", "rm", "rof", "rwk", "saq", "sdh", "seh", "sn", "so", "sq", "ss", "ssy", "st", "syr", "ta", "te", "teo", "tig", "tk", "tn", "tr", "ts", "ug", "uz", "ve", "vo", "vun", "wae", "xh", "xog"}, &Rule{ + addPluralRules(rules, []string{"af", "asa", "az", "bem", "bez", "bg", "brx", "ce", "cgg", "chr", "ckb", "dv", "ee", "el", "eo", "es", "eu", "fo", "fur", "gsw", "ha", "haw", "hu", "jgo", "jmc", "ka", "kaj", "kcg", "kk", "kkj", "kl", "ks", "ksb", "ku", "ky", "lb", "lg", "mas", "mgo", "ml", "mn", "nah", "nb", "nd", "ne", "nn", "nnh", "no", "nr", "ny", "nyn", "om", "or", "os", "pap", "ps", "rm", "rof", "rwk", "saq", "sd", "sdh", "seh", "sn", "so", "sq", "ss", "ssy", "st", "syr", "ta", "te", "teo", "tig", "tk", "tn", "tr", "ts", "ug", "uz", "ve", "vo", "vun", "wae", "xh", "xog"}, &Rule{ PluralForms: newPluralFormSet(One, Other), PluralFormFunc: func(ops *Operands) Form { // n = 1 @@ -122,9 +120,9 @@ func DefaultRules() map[language.Base]*Rule { addPluralRules(rules, []string{"mk"}, &Rule{ PluralForms: newPluralFormSet(One, Other), PluralFormFunc: func(ops *Operands) Form { - // v = 0 and i % 10 = 1 or f % 10 = 1 - if intEqualsAny(ops.V, 0) && intEqualsAny(ops.I%10, 1) || - intEqualsAny(ops.F%10, 1) { + // v = 0 and i % 10 = 1 and i % 100 != 11 or f % 10 = 1 and f % 100 != 11 + if intEqualsAny(ops.V, 0) && intEqualsAny(ops.I%10, 1) && !intEqualsAny(ops.I%100, 11) || + intEqualsAny(ops.F%10, 1) && !intEqualsAny(ops.F%100, 11) { return One } return Other diff --git a/v2/internal/plural/rule_gen_test.go b/v2/internal/plural/rule_gen_test.go index 569d29b0..ec0a3ce9 100644 --- a/v2/internal/plural/rule_gen_test.go +++ b/v2/internal/plural/rule_gen_test.go @@ -61,7 +61,7 @@ func TestPt(t *testing.T) { } } -func TestAstCaDeEnEtFiFyGlItJiNlSvSwUrYi(t *testing.T) { +func TestAstCaDeEnEtFiFyGlIoItJiNlPt_PTScnSvSwUrYi(t *testing.T) { var tests []pluralFormTest tests = appendIntegerTests(tests, One, []string{"1"}) @@ -69,7 +69,7 @@ func TestAstCaDeEnEtFiFyGlItJiNlSvSwUrYi(t *testing.T) { tests = appendIntegerTests(tests, Other, []string{"0", "2~16", "100", "1000", "10000", "100000", "1000000"}) tests = appendDecimalTests(tests, Other, []string{"0.0~1.5", "10.0", "100.0", "1000.0", "10000.0", "100000.0", "1000000.0"}) - locales := []string{"ast", "ca", "de", "en", "et", "fi", "fy", "gl", "it", "ji", "nl", "sv", "sw", "ur", "yi"} + locales := []string{"ast", "ca", "de", "en", "et", "fi", "fy", "gl", "io", "it", "ji", "nl", "pt_PT", "scn", "sv", "sw", "ur", "yi"} for _, locale := range locales { runTests(t, locale, tests) } @@ -120,7 +120,7 @@ func TestTzm(t *testing.T) { } } -func TestAfAsaAzBemBezBgBrxCeCggChrCkbDvEeElEoEsEuFoFurGswHaHawHuJgoJmcKaKajKcgKkKkjKlKsKsbKuKyLbLgMasMgoMlMnNahNbNdNeNnNnhNoNrNyNynOmOrOsPapPsRmRofRwkSaqSdhSehSnSoSqSsSsyStSyrTaTeTeoTigTkTnTrTsUgUzVeVoVunWaeXhXog(t *testing.T) { +func TestAfAsaAzBemBezBgBrxCeCggChrCkbDvEeElEoEsEuFoFurGswHaHawHuJgoJmcKaKajKcgKkKkjKlKsKsbKuKyLbLgMasMgoMlMnNahNbNdNeNnNnhNoNrNyNynOmOrOsPapPsRmRofRwkSaqSdSdhSehSnSoSqSsSsyStSyrTaTeTeoTigTkTnTrTsUgUzVeVoVunWaeXhXog(t *testing.T) { var tests []pluralFormTest tests = appendIntegerTests(tests, One, []string{"1"}) @@ -129,7 +129,7 @@ func TestAfAsaAzBemBezBgBrxCeCggChrCkbDvEeElEoEsEuFoFurGswHaHawHuJgoJmcKaKajKcgK tests = appendIntegerTests(tests, Other, []string{"0", "2~16", "100", "1000", "10000", "100000", "1000000"}) tests = appendDecimalTests(tests, Other, []string{"0.0~0.9", "1.1~1.6", "10.0", "100.0", "1000.0", "10000.0", "100000.0", "1000000.0"}) - locales := []string{"af", "asa", "az", "bem", "bez", "bg", "brx", "ce", "cgg", "chr", "ckb", "dv", "ee", "el", "eo", "es", "eu", "fo", "fur", "gsw", "ha", "haw", "hu", "jgo", "jmc", "ka", "kaj", "kcg", "kk", "kkj", "kl", "ks", "ksb", "ku", "ky", "lb", "lg", "mas", "mgo", "ml", "mn", "nah", "nb", "nd", "ne", "nn", "nnh", "no", "nr", "ny", "nyn", "om", "or", "os", "pap", "ps", "rm", "rof", "rwk", "saq", "sdh", "seh", "sn", "so", "sq", "ss", "ssy", "st", "syr", "ta", "te", "teo", "tig", "tk", "tn", "tr", "ts", "ug", "uz", "ve", "vo", "vun", "wae", "xh", "xog"} + locales := []string{"af", "asa", "az", "bem", "bez", "bg", "brx", "ce", "cgg", "chr", "ckb", "dv", "ee", "el", "eo", "es", "eu", "fo", "fur", "gsw", "ha", "haw", "hu", "jgo", "jmc", "ka", "kaj", "kcg", "kk", "kkj", "kl", "ks", "ksb", "ku", "ky", "lb", "lg", "mas", "mgo", "ml", "mn", "nah", "nb", "nd", "ne", "nn", "nnh", "no", "nr", "ny", "nyn", "om", "or", "os", "pap", "ps", "rm", "rof", "rwk", "saq", "sd", "sdh", "seh", "sn", "so", "sq", "ss", "ssy", "st", "syr", "ta", "te", "teo", "tig", "tk", "tn", "tr", "ts", "ug", "uz", "ve", "vo", "vun", "wae", "xh", "xog"} for _, locale := range locales { runTests(t, locale, tests) } @@ -168,10 +168,10 @@ func TestIs(t *testing.T) { func TestMk(t *testing.T) { var tests []pluralFormTest - tests = appendIntegerTests(tests, One, []string{"1", "11", "21", "31", "41", "51", "61", "71", "101", "1001"}) + tests = appendIntegerTests(tests, One, []string{"1", "21", "31", "41", "51", "61", "71", "81", "101", "1001"}) tests = appendDecimalTests(tests, One, []string{"0.1", "1.1", "2.1", "3.1", "4.1", "5.1", "6.1", "7.1", "10.1", "100.1", "1000.1"}) - tests = appendIntegerTests(tests, Other, []string{"0", "2~10", "12~17", "100", "1000", "10000", "100000", "1000000"}) + tests = appendIntegerTests(tests, Other, []string{"0", "2~16", "100", "1000", "10000", "100000", "1000000"}) tests = appendDecimalTests(tests, Other, []string{"0.0", "0.2~1.0", "1.2~1.7", "10.0", "100.0", "1000.0", "10000.0", "100000.0", "1000000.0"}) locales := []string{"mk"} diff --git a/v2/internal/plural/rule_test.go b/v2/internal/plural/rule_test.go index fe6503bb..85d26d93 100644 --- a/v2/internal/plural/rule_test.go +++ b/v2/internal/plural/rule_test.go @@ -18,8 +18,8 @@ func runTests(t *testing.T, pluralRuleID string, tests []pluralFormTest) { return } pluralRules := DefaultRules() - base := language.MustParseBase(pluralRuleID) - if rule := pluralRules[base]; rule != nil { + tag := language.MustParse(pluralRuleID) + if rule := pluralRules.Rule(tag); rule != nil { for _, test := range tests { ops, err := NewOperands(test.num) if err != nil { diff --git a/v2/internal/plural/rules.go b/v2/internal/plural/rules.go new file mode 100644 index 00000000..373372a2 --- /dev/null +++ b/v2/internal/plural/rules.go @@ -0,0 +1,20 @@ +package plural + +import "golang.org/x/text/language" + +// Rules is a set of plural rules by language tag. +type Rules map[language.Tag]*Rule + +// Rule returns the closest matching plural rule for the language tag +// or nil if no rule could be found. +func (r Rules) Rule(tag language.Tag) *Rule { + for { + if rule := r[tag]; rule != nil { + return rule + } + tag = tag.Parent() + if tag.IsRoot() { + return nil + } + } +} diff --git a/v2/internal/plural/rules_test.go b/v2/internal/plural/rules_test.go new file mode 100644 index 00000000..b7666d67 --- /dev/null +++ b/v2/internal/plural/rules_test.go @@ -0,0 +1,61 @@ +package plural + +import ( + "testing" + + "golang.org/x/text/language" +) + +func TestRules(t *testing.T) { + expectedRule := &Rule{} + + testCases := []struct { + name string + rules Rules + tag language.Tag + rule *Rule + }{ + { + name: "exact match", + rules: Rules{ + language.English: expectedRule, + language.Spanish: &Rule{}, + }, + tag: language.English, + rule: expectedRule, + }, + { + name: "inexact match", + rules: Rules{ + language.English: expectedRule, + }, + tag: language.AmericanEnglish, + rule: expectedRule, + }, + { + name: "portuguese doesn't match european portuguese", + rules: Rules{ + language.EuropeanPortuguese: &Rule{}, + }, + tag: language.Portuguese, + rule: nil, + }, + { + name: "european portuguese preferred", + rules: Rules{ + language.Portuguese: &Rule{}, + language.EuropeanPortuguese: expectedRule, + }, + tag: language.EuropeanPortuguese, + rule: expectedRule, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + if rule := testCase.rules.Rule(testCase.tag); rule != testCase.rule { + panic(rule) + } + }) + } +}