mirror of
https://github.com/cwinfo/matterbridge.git
synced 2024-12-27 08:55:39 +00:00
603 lines
15 KiB
Go
603 lines
15 KiB
Go
// Copyright 2014 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
// +build ignore
|
|
|
|
// Generator for display name tables.
|
|
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"flag"
|
|
"fmt"
|
|
"log"
|
|
"reflect"
|
|
"sort"
|
|
"strings"
|
|
|
|
"golang.org/x/text/internal/gen"
|
|
"golang.org/x/text/language"
|
|
"golang.org/x/text/unicode/cldr"
|
|
)
|
|
|
|
var (
|
|
test = flag.Bool("test", false,
|
|
"test existing tables; can be used to compare web data with package data.")
|
|
outputFile = flag.String("output", "tables.go", "output file")
|
|
|
|
stats = flag.Bool("stats", false, "prints statistics to stderr")
|
|
|
|
short = flag.Bool("short", false, `Use "short" alternatives, when available.`)
|
|
draft = flag.String("draft",
|
|
"contributed",
|
|
`Minimal draft requirements (approved, contributed, provisional, unconfirmed).`)
|
|
pkg = flag.String("package",
|
|
"display",
|
|
"the name of the package in which the generated file is to be included")
|
|
|
|
tags = newTagSet("tags",
|
|
[]language.Tag{},
|
|
"space-separated list of tags to include or empty for all")
|
|
dict = newTagSet("dict",
|
|
dictTags(),
|
|
"space-separated list or tags for which to include a Dictionary. "+
|
|
`"" means the common list from go.text/language.`)
|
|
)
|
|
|
|
func dictTags() (tag []language.Tag) {
|
|
// TODO: replace with language.Common.Tags() once supported.
|
|
const str = "af am ar ar-001 az bg bn ca cs da de el en en-US en-GB " +
|
|
"es es-ES es-419 et fa fi fil fr fr-CA gu he hi hr hu hy id is it ja " +
|
|
"ka kk km kn ko ky lo lt lv mk ml mn mr ms my ne nl no pa pl pt pt-BR " +
|
|
"pt-PT ro ru si sk sl sq sr sr-Latn sv sw ta te th tr uk ur uz vi " +
|
|
"zh zh-Hans zh-Hant zu"
|
|
|
|
for _, s := range strings.Split(str, " ") {
|
|
tag = append(tag, language.MustParse(s))
|
|
}
|
|
return tag
|
|
}
|
|
|
|
func main() {
|
|
gen.Init()
|
|
|
|
// Read the CLDR zip file.
|
|
r := gen.OpenCLDRCoreZip()
|
|
defer r.Close()
|
|
|
|
d := &cldr.Decoder{}
|
|
d.SetDirFilter("main", "supplemental")
|
|
d.SetSectionFilter("localeDisplayNames")
|
|
data, err := d.DecodeZip(r)
|
|
if err != nil {
|
|
log.Fatalf("DecodeZip: %v", err)
|
|
}
|
|
|
|
w := gen.NewCodeWriter()
|
|
defer w.WriteGoFile(*outputFile, "display")
|
|
|
|
gen.WriteCLDRVersion(w)
|
|
|
|
b := builder{
|
|
w: w,
|
|
data: data,
|
|
group: make(map[string]*group),
|
|
}
|
|
b.generate()
|
|
}
|
|
|
|
const tagForm = language.All
|
|
|
|
// tagSet is used to parse command line flags of tags. It implements the
|
|
// flag.Value interface.
|
|
type tagSet map[language.Tag]bool
|
|
|
|
func newTagSet(name string, tags []language.Tag, usage string) tagSet {
|
|
f := tagSet(make(map[language.Tag]bool))
|
|
for _, t := range tags {
|
|
f[t] = true
|
|
}
|
|
flag.Var(f, name, usage)
|
|
return f
|
|
}
|
|
|
|
// String implements the String method of the flag.Value interface.
|
|
func (f tagSet) String() string {
|
|
tags := []string{}
|
|
for t := range f {
|
|
tags = append(tags, t.String())
|
|
}
|
|
sort.Strings(tags)
|
|
return strings.Join(tags, " ")
|
|
}
|
|
|
|
// Set implements Set from the flag.Value interface.
|
|
func (f tagSet) Set(s string) error {
|
|
if s != "" {
|
|
for _, s := range strings.Split(s, " ") {
|
|
if s != "" {
|
|
tag, err := tagForm.Parse(s)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
f[tag] = true
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (f tagSet) contains(t language.Tag) bool {
|
|
if len(f) == 0 {
|
|
return true
|
|
}
|
|
return f[t]
|
|
}
|
|
|
|
// builder is used to create all tables with display name information.
|
|
type builder struct {
|
|
w *gen.CodeWriter
|
|
|
|
data *cldr.CLDR
|
|
|
|
fromLocs []string
|
|
|
|
// destination tags for the current locale.
|
|
toTags []string
|
|
toTagIndex map[string]int
|
|
|
|
// list of supported tags
|
|
supported []language.Tag
|
|
|
|
// key-value pairs per group
|
|
group map[string]*group
|
|
|
|
// statistics
|
|
sizeIndex int // total size of all indexes of headers
|
|
sizeData int // total size of all data of headers
|
|
totalSize int
|
|
}
|
|
|
|
type group struct {
|
|
// Maps from a given language to the Namer data for this language.
|
|
lang map[language.Tag]keyValues
|
|
headers []header
|
|
|
|
toTags []string
|
|
threeStart int
|
|
fourPlusStart int
|
|
}
|
|
|
|
// set sets the typ to the name for locale loc.
|
|
func (g *group) set(t language.Tag, typ, name string) {
|
|
kv := g.lang[t]
|
|
if kv == nil {
|
|
kv = make(keyValues)
|
|
g.lang[t] = kv
|
|
}
|
|
if kv[typ] == "" {
|
|
kv[typ] = name
|
|
}
|
|
}
|
|
|
|
type keyValues map[string]string
|
|
|
|
type header struct {
|
|
tag language.Tag
|
|
data string
|
|
index []uint16
|
|
}
|
|
|
|
var versionInfo = `// Version is deprecated. Use CLDRVersion.
|
|
const Version = %#v
|
|
|
|
`
|
|
|
|
var self = language.MustParse("mul")
|
|
|
|
// generate builds and writes all tables.
|
|
func (b *builder) generate() {
|
|
fmt.Fprintf(b.w, versionInfo, cldr.Version)
|
|
|
|
b.filter()
|
|
b.setData("lang", func(g *group, loc language.Tag, ldn *cldr.LocaleDisplayNames) {
|
|
if ldn.Languages != nil {
|
|
for _, v := range ldn.Languages.Language {
|
|
lang := v.Type
|
|
if lang == "root" {
|
|
// We prefer the data from "und"
|
|
// TODO: allow both the data for root and und somehow.
|
|
continue
|
|
}
|
|
tag := tagForm.MustParse(lang)
|
|
if tags.contains(tag) {
|
|
g.set(loc, tag.String(), v.Data())
|
|
}
|
|
}
|
|
}
|
|
})
|
|
b.setData("script", func(g *group, loc language.Tag, ldn *cldr.LocaleDisplayNames) {
|
|
if ldn.Scripts != nil {
|
|
for _, v := range ldn.Scripts.Script {
|
|
code := language.MustParseScript(v.Type)
|
|
if code.IsPrivateUse() { // Qaaa..Qabx
|
|
// TODO: data currently appears to be very meager.
|
|
// Reconsider if we have data for English.
|
|
if loc == language.English {
|
|
log.Fatal("Consider including data for private use scripts.")
|
|
}
|
|
continue
|
|
}
|
|
g.set(loc, code.String(), v.Data())
|
|
}
|
|
}
|
|
})
|
|
b.setData("region", func(g *group, loc language.Tag, ldn *cldr.LocaleDisplayNames) {
|
|
if ldn.Territories != nil {
|
|
for _, v := range ldn.Territories.Territory {
|
|
g.set(loc, language.MustParseRegion(v.Type).String(), v.Data())
|
|
}
|
|
}
|
|
})
|
|
|
|
b.makeSupported()
|
|
|
|
b.writeParents()
|
|
|
|
b.writeGroup("lang")
|
|
b.writeGroup("script")
|
|
b.writeGroup("region")
|
|
|
|
b.w.WriteConst("numSupported", len(b.supported))
|
|
buf := bytes.Buffer{}
|
|
for _, tag := range b.supported {
|
|
fmt.Fprint(&buf, tag.String(), "|")
|
|
}
|
|
b.w.WriteConst("supported", buf.String())
|
|
|
|
b.writeDictionaries()
|
|
|
|
b.supported = []language.Tag{self}
|
|
|
|
// Compute the names of locales in their own language. Some of these names
|
|
// may be specified in their parent locales. We iterate the maximum depth
|
|
// of the parent three times to match successive parents of tags until a
|
|
// possible match is found.
|
|
for i := 0; i < 4; i++ {
|
|
b.setData("self", func(g *group, tag language.Tag, ldn *cldr.LocaleDisplayNames) {
|
|
parent := tag
|
|
if b, s, r := tag.Raw(); i > 0 && (s != language.Script{} && r == language.Region{}) {
|
|
parent, _ = language.Raw.Compose(b)
|
|
}
|
|
if ldn.Languages != nil {
|
|
for _, v := range ldn.Languages.Language {
|
|
key := tagForm.MustParse(v.Type)
|
|
saved := key
|
|
if key == parent {
|
|
g.set(self, tag.String(), v.Data())
|
|
}
|
|
for k := 0; k < i; k++ {
|
|
key = key.Parent()
|
|
}
|
|
if key == tag {
|
|
g.set(self, saved.String(), v.Data()) // set does not overwrite a value.
|
|
}
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
b.writeGroup("self")
|
|
}
|
|
|
|
func (b *builder) setData(name string, f func(*group, language.Tag, *cldr.LocaleDisplayNames)) {
|
|
b.sizeIndex = 0
|
|
b.sizeData = 0
|
|
b.toTags = nil
|
|
b.fromLocs = nil
|
|
b.toTagIndex = make(map[string]int)
|
|
|
|
g := b.group[name]
|
|
if g == nil {
|
|
g = &group{lang: make(map[language.Tag]keyValues)}
|
|
b.group[name] = g
|
|
}
|
|
for _, loc := range b.data.Locales() {
|
|
// We use RawLDML instead of LDML as we are managing our own inheritance
|
|
// in this implementation.
|
|
ldml := b.data.RawLDML(loc)
|
|
|
|
// We do not support the POSIX variant (it is not a supported BCP 47
|
|
// variant). This locale also doesn't happen to contain any data, so
|
|
// we'll skip it by checking for this.
|
|
tag, err := tagForm.Parse(loc)
|
|
if err != nil {
|
|
if ldml.LocaleDisplayNames != nil {
|
|
log.Fatalf("setData: %v", err)
|
|
}
|
|
continue
|
|
}
|
|
if ldml.LocaleDisplayNames != nil && tags.contains(tag) {
|
|
f(g, tag, ldml.LocaleDisplayNames)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (b *builder) filter() {
|
|
filter := func(s *cldr.Slice) {
|
|
if *short {
|
|
s.SelectOnePerGroup("alt", []string{"short", ""})
|
|
} else {
|
|
s.SelectOnePerGroup("alt", []string{"stand-alone", ""})
|
|
}
|
|
d, err := cldr.ParseDraft(*draft)
|
|
if err != nil {
|
|
log.Fatalf("filter: %v", err)
|
|
}
|
|
s.SelectDraft(d)
|
|
}
|
|
for _, loc := range b.data.Locales() {
|
|
if ldn := b.data.RawLDML(loc).LocaleDisplayNames; ldn != nil {
|
|
if ldn.Languages != nil {
|
|
s := cldr.MakeSlice(&ldn.Languages.Language)
|
|
if filter(&s); len(ldn.Languages.Language) == 0 {
|
|
ldn.Languages = nil
|
|
}
|
|
}
|
|
if ldn.Scripts != nil {
|
|
s := cldr.MakeSlice(&ldn.Scripts.Script)
|
|
if filter(&s); len(ldn.Scripts.Script) == 0 {
|
|
ldn.Scripts = nil
|
|
}
|
|
}
|
|
if ldn.Territories != nil {
|
|
s := cldr.MakeSlice(&ldn.Territories.Territory)
|
|
if filter(&s); len(ldn.Territories.Territory) == 0 {
|
|
ldn.Territories = nil
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// makeSupported creates a list of all supported locales.
|
|
func (b *builder) makeSupported() {
|
|
// tags across groups
|
|
for _, g := range b.group {
|
|
for t, _ := range g.lang {
|
|
b.supported = append(b.supported, t)
|
|
}
|
|
}
|
|
b.supported = b.supported[:unique(tagsSorter(b.supported))]
|
|
|
|
}
|
|
|
|
type tagsSorter []language.Tag
|
|
|
|
func (a tagsSorter) Len() int { return len(a) }
|
|
func (a tagsSorter) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
|
func (a tagsSorter) Less(i, j int) bool { return a[i].String() < a[j].String() }
|
|
|
|
func (b *builder) writeGroup(name string) {
|
|
g := b.group[name]
|
|
|
|
for _, kv := range g.lang {
|
|
for t, _ := range kv {
|
|
g.toTags = append(g.toTags, t)
|
|
}
|
|
}
|
|
g.toTags = g.toTags[:unique(tagsBySize(g.toTags))]
|
|
|
|
// Allocate header per supported value.
|
|
g.headers = make([]header, len(b.supported))
|
|
for i, sup := range b.supported {
|
|
kv, ok := g.lang[sup]
|
|
if !ok {
|
|
g.headers[i].tag = sup
|
|
continue
|
|
}
|
|
data := []byte{}
|
|
index := make([]uint16, len(g.toTags), len(g.toTags)+1)
|
|
for j, t := range g.toTags {
|
|
index[j] = uint16(len(data))
|
|
data = append(data, kv[t]...)
|
|
}
|
|
index = append(index, uint16(len(data)))
|
|
|
|
// Trim the tail of the index.
|
|
// TODO: indexes can be reduced in size quite a bit more.
|
|
n := len(index)
|
|
for ; n >= 2 && index[n-2] == index[n-1]; n-- {
|
|
}
|
|
index = index[:n]
|
|
|
|
// Workaround for a bug in CLDR 26.
|
|
// See http://unicode.org/cldr/trac/ticket/8042.
|
|
if cldr.Version == "26" && sup.String() == "hsb" {
|
|
data = bytes.Replace(data, []byte{'"'}, nil, 1)
|
|
}
|
|
g.headers[i] = header{sup, string(data), index}
|
|
}
|
|
g.writeTable(b.w, name)
|
|
}
|
|
|
|
type tagsBySize []string
|
|
|
|
func (l tagsBySize) Len() int { return len(l) }
|
|
func (l tagsBySize) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
|
|
func (l tagsBySize) Less(i, j int) bool {
|
|
a, b := l[i], l[j]
|
|
// Sort single-tag entries based on size first. Otherwise alphabetic.
|
|
if len(a) != len(b) && (len(a) <= 4 || len(b) <= 4) {
|
|
return len(a) < len(b)
|
|
}
|
|
return a < b
|
|
}
|
|
|
|
// parentIndices returns slice a of len(tags) where tags[a[i]] is the parent
|
|
// of tags[i].
|
|
func parentIndices(tags []language.Tag) []int16 {
|
|
index := make(map[language.Tag]int16)
|
|
for i, t := range tags {
|
|
index[t] = int16(i)
|
|
}
|
|
|
|
// Construct default parents.
|
|
parents := make([]int16, len(tags))
|
|
for i, t := range tags {
|
|
parents[i] = -1
|
|
for t = t.Parent(); t != language.Und; t = t.Parent() {
|
|
if j, ok := index[t]; ok {
|
|
parents[i] = j
|
|
break
|
|
}
|
|
}
|
|
}
|
|
return parents
|
|
}
|
|
|
|
func (b *builder) writeParents() {
|
|
parents := parentIndices(b.supported)
|
|
fmt.Fprintf(b.w, "var parents = ")
|
|
b.w.WriteArray(parents)
|
|
}
|
|
|
|
// writeKeys writes keys to a special index used by the display package.
|
|
// tags are assumed to be sorted by length.
|
|
func writeKeys(w *gen.CodeWriter, name string, keys []string) {
|
|
w.Size += int(3 * reflect.TypeOf("").Size())
|
|
w.WriteComment("Number of keys: %d", len(keys))
|
|
fmt.Fprintf(w, "var (\n\t%sIndex = tagIndex{\n", name)
|
|
for i := 2; i <= 4; i++ {
|
|
sub := []string{}
|
|
for _, t := range keys {
|
|
if len(t) != i {
|
|
break
|
|
}
|
|
sub = append(sub, t)
|
|
}
|
|
s := strings.Join(sub, "")
|
|
w.WriteString(s)
|
|
fmt.Fprintf(w, ",\n")
|
|
keys = keys[len(sub):]
|
|
}
|
|
fmt.Fprintln(w, "\t}")
|
|
if len(keys) > 0 {
|
|
w.Size += int(reflect.TypeOf([]string{}).Size())
|
|
fmt.Fprintf(w, "\t%sTagsLong = ", name)
|
|
w.WriteSlice(keys)
|
|
}
|
|
fmt.Fprintln(w, ")\n")
|
|
}
|
|
|
|
// identifier creates an identifier from the given tag.
|
|
func identifier(t language.Tag) string {
|
|
return strings.Replace(t.String(), "-", "", -1)
|
|
}
|
|
|
|
func (h *header) writeEntry(w *gen.CodeWriter, name string) {
|
|
if len(dict) > 0 && dict.contains(h.tag) {
|
|
fmt.Fprintf(w, "\t{ // %s\n", h.tag)
|
|
fmt.Fprintf(w, "\t\t%[1]s%[2]sStr,\n\t\t%[1]s%[2]sIdx,\n", identifier(h.tag), name)
|
|
fmt.Fprintln(w, "\t},")
|
|
} else if len(h.data) == 0 {
|
|
fmt.Fprintln(w, "\t\t{}, //", h.tag)
|
|
} else {
|
|
fmt.Fprintf(w, "\t{ // %s\n", h.tag)
|
|
w.WriteString(h.data)
|
|
fmt.Fprintln(w, ",")
|
|
w.WriteSlice(h.index)
|
|
fmt.Fprintln(w, ",\n\t},")
|
|
}
|
|
}
|
|
|
|
// write the data for the given header as single entries. The size for this data
|
|
// was already accounted for in writeEntry.
|
|
func (h *header) writeSingle(w *gen.CodeWriter, name string) {
|
|
if len(dict) > 0 && dict.contains(h.tag) {
|
|
tag := identifier(h.tag)
|
|
w.WriteConst(tag+name+"Str", h.data)
|
|
|
|
// Note that we create a slice instead of an array. If we use an array
|
|
// we need to refer to it as a[:] in other tables, which will cause the
|
|
// array to always be included by the linker. See Issue 7651.
|
|
w.WriteVar(tag+name+"Idx", h.index)
|
|
}
|
|
}
|
|
|
|
// WriteTable writes an entry for a single Namer.
|
|
func (g *group) writeTable(w *gen.CodeWriter, name string) {
|
|
start := w.Size
|
|
writeKeys(w, name, g.toTags)
|
|
w.Size += len(g.headers) * int(reflect.ValueOf(g.headers[0]).Type().Size())
|
|
|
|
fmt.Fprintf(w, "var %sHeaders = [%d]header{\n", name, len(g.headers))
|
|
|
|
title := strings.Title(name)
|
|
for _, h := range g.headers {
|
|
h.writeEntry(w, title)
|
|
}
|
|
fmt.Fprintln(w, "}\n")
|
|
|
|
for _, h := range g.headers {
|
|
h.writeSingle(w, title)
|
|
}
|
|
n := w.Size - start
|
|
fmt.Fprintf(w, "// Total size for %s: %d bytes (%d KB)\n\n", name, n, n/1000)
|
|
}
|
|
|
|
func (b *builder) writeDictionaries() {
|
|
fmt.Fprintln(b.w, "// Dictionary entries of frequent languages")
|
|
fmt.Fprintln(b.w, "var (")
|
|
parents := parentIndices(b.supported)
|
|
|
|
for i, t := range b.supported {
|
|
if dict.contains(t) {
|
|
ident := identifier(t)
|
|
fmt.Fprintf(b.w, "\t%s = Dictionary{ // %s\n", ident, t)
|
|
if p := parents[i]; p == -1 {
|
|
fmt.Fprintln(b.w, "\t\tnil,")
|
|
} else {
|
|
fmt.Fprintf(b.w, "\t\t&%s,\n", identifier(b.supported[p]))
|
|
}
|
|
fmt.Fprintf(b.w, "\t\theader{%[1]sLangStr, %[1]sLangIdx},\n", ident)
|
|
fmt.Fprintf(b.w, "\t\theader{%[1]sScriptStr, %[1]sScriptIdx},\n", ident)
|
|
fmt.Fprintf(b.w, "\t\theader{%[1]sRegionStr, %[1]sRegionIdx},\n", ident)
|
|
fmt.Fprintln(b.w, "\t}")
|
|
}
|
|
}
|
|
fmt.Fprintln(b.w, ")")
|
|
|
|
var s string
|
|
var a []uint16
|
|
sz := reflect.TypeOf(s).Size()
|
|
sz += reflect.TypeOf(a).Size()
|
|
sz *= 3
|
|
sz += reflect.TypeOf(&a).Size()
|
|
n := int(sz) * len(dict)
|
|
fmt.Fprintf(b.w, "// Total size for %d entries: %d bytes (%d KB)\n\n", len(dict), n, n/1000)
|
|
|
|
b.w.Size += n
|
|
}
|
|
|
|
// unique sorts the given lists and removes duplicate entries by swapping them
|
|
// past position k, where k is the number of unique values. It returns k.
|
|
func unique(a sort.Interface) int {
|
|
if a.Len() == 0 {
|
|
return 0
|
|
}
|
|
sort.Sort(a)
|
|
k := 1
|
|
for i := 1; i < a.Len(); i++ {
|
|
if a.Less(k-1, i) {
|
|
if k != i {
|
|
a.Swap(k, i)
|
|
}
|
|
k++
|
|
}
|
|
}
|
|
return k
|
|
}
|