Add vendor folder to git

vendor/golang.org/x/text/internal/colltab/colltab.go

@@ -0,0 +1,105 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package colltab contains functionality related to collation tables.
+// It is only to be used by the collate and search packages.
+package colltab // import "golang.org/x/text/internal/colltab"
+
+import (
+	"sort"
+
+	"golang.org/x/text/language"
+)
+
+// MatchLang finds the index of t in tags, using a matching algorithm used for
+// collation and search. tags[0] must be language.Und, the remaining tags should
+// be sorted alphabetically.
+//
+// Language matching for collation and search is different from the matching
+// defined by language.Matcher: the (inferred) base language must be an exact
+// match for the relevant fields. For example, "gsw" should not match "de".
+// Also the parent relation is different, as a parent may have a different
+// script. So usually the parent of zh-Hant is und, whereas for MatchLang it is
+// zh.
+func MatchLang(t language.Tag, tags []language.Tag) int {
+	// Canonicalize the values, including collapsing macro languages.
+	t, _ = language.All.Canonicalize(t)
+
+	base, conf := t.Base()
+	// Estimate the base language, but only use high-confidence values.
+	if conf < language.High {
+		// The root locale supports "search" and "standard". We assume that any
+		// implementation will only use one of both.
+		return 0
+	}
+
+	// Maximize base and script and normalize the tag.
+	if _, s, r := t.Raw(); (r != language.Region{}) {
+		p, _ := language.Raw.Compose(base, s, r)
+		// Taking the parent forces the script to be maximized.
+		p = p.Parent()
+		// Add back region and extensions.
+		t, _ = language.Raw.Compose(p, r, t.Extensions())
+	} else {
+		// Set the maximized base language.
+		t, _ = language.Raw.Compose(base, s, t.Extensions())
+	}
+
+	// Find start index of the language tag.
+	start := 1 + sort.Search(len(tags)-1, func(i int) bool {
+		b, _, _ := tags[i+1].Raw()
+		return base.String() <= b.String()
+	})
+	if start < len(tags) {
+		if b, _, _ := tags[start].Raw(); b != base {
+			return 0
+		}
+	}
+
+	// Besides the base language, script and region, only the collation type and
+	// the custom variant defined in the 'u' extension are used to distinguish a
+	// locale.
+	// Strip all variants and extensions and add back the custom variant.
+	tdef, _ := language.Raw.Compose(t.Raw())
+	tdef, _ = tdef.SetTypeForKey("va", t.TypeForKey("va"))
+
+	// First search for a specialized collation type, if present.
+	try := []language.Tag{tdef}
+	if co := t.TypeForKey("co"); co != "" {
+		tco, _ := tdef.SetTypeForKey("co", co)
+		try = []language.Tag{tco, tdef}
+	}
+
+	for _, tx := range try {
+		for ; tx != language.Und; tx = parent(tx) {
+			for i, t := range tags[start:] {
+				if b, _, _ := t.Raw(); b != base {
+					break
+				}
+				if tx == t {
+					return start + i
+				}
+			}
+		}
+	}
+	return 0
+}
+
+// parent computes the structural parent. This means inheritance may change
+// script. So, unlike the CLDR parent, parent(zh-Hant) == zh.
+func parent(t language.Tag) language.Tag {
+	if t.TypeForKey("va") != "" {
+		t, _ = t.SetTypeForKey("va", "")
+		return t
+	}
+	result := language.Und
+	if b, s, r := t.Raw(); (r != language.Region{}) {
+		result, _ = language.Raw.Compose(b, s, t.Extensions())
+	} else if (s != language.Script{}) {
+		result, _ = language.Raw.Compose(b, t.Extensions())
+	} else if (b != language.Base{}) {
+		result, _ = language.Raw.Compose(t.Extensions())
+	}
+	return result
+}

vendor/golang.org/x/text/internal/colltab/colltab_test.go

@@ -0,0 +1,64 @@
+package colltab
+
+import (
+	"testing"
+
+	"golang.org/x/text/language"
+)
+
+func TestMatchLang(t *testing.T) {
+	tags := []language.Tag{
+		0:  language.Und,
+		1:  language.MustParse("bs"),
+		2:  language.German,
+		3:  language.English,
+		4:  language.AmericanEnglish,
+		5:  language.MustParse("en-US-u-va-posix"),
+		6:  language.Portuguese,
+		7:  language.Serbian,
+		8:  language.MustParse("sr-Latn"),
+		9:  language.Chinese,
+		10: language.MustParse("zh-u-co-stroke"),
+		11: language.MustParse("zh-Hant-u-co-pinyin"),
+		12: language.TraditionalChinese,
+	}
+	for i, tc := range []struct {
+		x int
+		t language.Tag
+	}{
+		{0, language.Und},
+		{0, language.Persian}, // Default to first element when no match.
+		{3, language.English},
+		{4, language.AmericanEnglish},
+		{5, language.MustParse("en-US-u-va-posix")},   // Ext. variant match.
+		{4, language.MustParse("en-US-u-va-noposix")}, // Ext. variant mismatch.
+		{3, language.MustParse("en-UK-u-va-noposix")}, // Ext. variant mismatch.
+		{7, language.Serbian},
+		{0, language.Croatian},             // Don't match to close language!
+		{0, language.MustParse("gsw")},     // Don't match to close language!
+		{1, language.MustParse("bs-Cyrl")}, // Odd, but correct.
+		{1, language.MustParse("bs-Latn")}, // Estimated script drops.
+		{8, language.MustParse("sr-Latn")},
+		{9, language.Chinese},
+		{9, language.SimplifiedChinese},
+		{12, language.TraditionalChinese},
+		{11, language.MustParse("zh-Hant-u-co-pinyin")},
+		// TODO: should this be 12? Either inherited value (10) or default is
+		// fine in this case, though. Other locales are not affected.
+		{10, language.MustParse("zh-Hant-u-co-stroke")},
+		// There is no "phonebk" sorting order for zh-Hant, so use default.
+		{12, language.MustParse("zh-Hant-u-co-phonebk")},
+		{10, language.MustParse("zh-u-co-stroke")},
+		{12, language.MustParse("und-TW")},     // Infer script and language.
+		{12, language.MustParse("und-HK")},     // Infer script and language.
+		{6, language.MustParse("und-BR")},      // Infer script and language.
+		{6, language.MustParse("und-PT")},      // Infer script and language.
+		{2, language.MustParse("und-Latn-DE")}, // Infer language.
+		{0, language.MustParse("und-Jpan-BR")}, // Infers "ja", so no match.
+		{0, language.MustParse("zu")},          // No match past index.
+	} {
+		if x := MatchLang(tc.t, tags); x != tc.x {
+			t.Errorf("%d: MatchLang(%q, tags) = %d; want %d", i, tc.t, x, tc.x)
+		}
+	}
+}

vendor/golang.org/x/text/internal/colltab/contract.go

@@ -0,0 +1,145 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package colltab
+
+import "unicode/utf8"
+
+// For a description of contractTrieSet, see text/collate/build/contract.go.
+
+type contractTrieSet []struct{ l, h, n, i uint8 }
+
+// ctScanner is used to match a trie to an input sequence.
+// A contraction may match a non-contiguous sequence of bytes in an input string.
+// For example, if there is a contraction for <a, combining_ring>, it should match
+// the sequence <a, combining_cedilla, combining_ring>, as combining_cedilla does
+// not block combining_ring.
+// ctScanner does not automatically skip over non-blocking non-starters, but rather
+// retains the state of the last match and leaves it up to the user to continue
+// the match at the appropriate points.
+type ctScanner struct {
+	states contractTrieSet
+	s      []byte
+	n      int
+	index  int
+	pindex int
+	done   bool
+}
+
+type ctScannerString struct {
+	states contractTrieSet
+	s      string
+	n      int
+	index  int
+	pindex int
+	done   bool
+}
+
+func (t contractTrieSet) scanner(index, n int, b []byte) ctScanner {
+	return ctScanner{s: b, states: t[index:], n: n}
+}
+
+func (t contractTrieSet) scannerString(index, n int, str string) ctScannerString {
+	return ctScannerString{s: str, states: t[index:], n: n}
+}
+
+// result returns the offset i and bytes consumed p so far.  If no suffix
+// matched, i and p will be 0.
+func (s *ctScanner) result() (i, p int) {
+	return s.index, s.pindex
+}
+
+func (s *ctScannerString) result() (i, p int) {
+	return s.index, s.pindex
+}
+
+const (
+	final   = 0
+	noIndex = 0xFF
+)
+
+// scan matches the longest suffix at the current location in the input
+// and returns the number of bytes consumed.
+func (s *ctScanner) scan(p int) int {
+	pr := p // the p at the rune start
+	str := s.s
+	states, n := s.states, s.n
+	for i := 0; i < n && p < len(str); {
+		e := states[i]
+		c := str[p]
+		// TODO: a significant number of contractions are of a form that
+		// cannot match discontiguous UTF-8 in a normalized string. We could let
+		// a negative value of e.n mean that we can set s.done = true and avoid
+		// the need for additional matches.
+		if c >= e.l {
+			if e.l == c {
+				p++
+				if e.i != noIndex {
+					s.index = int(e.i)
+					s.pindex = p
+				}
+				if e.n != final {
+					i, states, n = 0, states[int(e.h)+n:], int(e.n)
+					if p >= len(str) || utf8.RuneStart(str[p]) {
+						s.states, s.n, pr = states, n, p
+					}
+				} else {
+					s.done = true
+					return p
+				}
+				continue
+			} else if e.n == final && c <= e.h {
+				p++
+				s.done = true
+				s.index = int(c-e.l) + int(e.i)
+				s.pindex = p
+				return p
+			}
+		}
+		i++
+	}
+	return pr
+}
+
+// scan is a verbatim copy of ctScanner.scan.
+func (s *ctScannerString) scan(p int) int {
+	pr := p // the p at the rune start
+	str := s.s
+	states, n := s.states, s.n
+	for i := 0; i < n && p < len(str); {
+		e := states[i]
+		c := str[p]
+		// TODO: a significant number of contractions are of a form that
+		// cannot match discontiguous UTF-8 in a normalized string. We could let
+		// a negative value of e.n mean that we can set s.done = true and avoid
+		// the need for additional matches.
+		if c >= e.l {
+			if e.l == c {
+				p++
+				if e.i != noIndex {
+					s.index = int(e.i)
+					s.pindex = p
+				}
+				if e.n != final {
+					i, states, n = 0, states[int(e.h)+n:], int(e.n)
+					if p >= len(str) || utf8.RuneStart(str[p]) {
+						s.states, s.n, pr = states, n, p
+					}
+				} else {
+					s.done = true
+					return p
+				}
+				continue
+			} else if e.n == final && c <= e.h {
+				p++
+				s.done = true
+				s.index = int(c-e.l) + int(e.i)
+				s.pindex = p
+				return p
+			}
+		}
+		i++
+	}
+	return pr
+}

vendor/golang.org/x/text/internal/colltab/contract_test.go

@@ -0,0 +1,135 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package colltab
+
+import (
+	"testing"
+)
+
+type lookupStrings struct {
+	str    string
+	offset int
+	n      int // bytes consumed from input
+}
+
+var lookupTests = []struct {
+	lookup []lookupStrings
+	n      int
+	tries  contractTrieSet
+}{
+	{
+		[]lookupStrings{
+			{"abc", 1, 3},
+			{"a", 0, 0},
+			{"b", 0, 0},
+			{"c", 0, 0},
+			{"d", 0, 0},
+		},
+		1,
+		contractTrieSet{
+			{'a', 0, 1, 0xFF},
+			{'b', 0, 1, 0xFF},
+			{'c', 'c', 0, 1},
+		},
+	},
+	{
+		[]lookupStrings{
+			{"abc", 1, 3},
+			{"abd", 2, 3},
+			{"abe", 3, 3},
+			{"a", 0, 0},
+			{"ab", 0, 0},
+			{"d", 0, 0},
+			{"f", 0, 0},
+		},
+		1,
+		contractTrieSet{
+			{'a', 0, 1, 0xFF},
+			{'b', 0, 1, 0xFF},
+			{'c', 'e', 0, 1},
+		},
+	},
+	{
+		[]lookupStrings{
+			{"abc", 1, 3},
+			{"ab", 2, 2},
+			{"a", 3, 1},
+			{"abcd", 1, 3},
+			{"abe", 2, 2},
+		},
+		1,
+		contractTrieSet{
+			{'a', 0, 1, 3},
+			{'b', 0, 1, 2},
+			{'c', 'c', 0, 1},
+		},
+	},
+	{
+		[]lookupStrings{
+			{"abc", 1, 3},
+			{"abd", 2, 3},
+			{"ab", 3, 2},
+			{"ac", 4, 2},
+			{"a", 5, 1},
+			{"b", 6, 1},
+			{"ba", 6, 1},
+		},
+		2,
+		contractTrieSet{
+			{'b', 'b', 0, 6},
+			{'a', 0, 2, 5},
+			{'c', 'c', 0, 4},
+			{'b', 0, 1, 3},
+			{'c', 'd', 0, 1},
+		},
+	},
+	{
+		[]lookupStrings{
+			{"bcde", 2, 4},
+			{"bc", 7, 2},
+			{"ab", 6, 2},
+			{"bcd", 5, 3},
+			{"abcd", 1, 4},
+			{"abc", 4, 3},
+			{"bcdf", 3, 4},
+		},
+		2,
+		contractTrieSet{
+			{'b', 3, 1, 0xFF},
+			{'a', 0, 1, 0xFF},
+			{'b', 0, 1, 6},
+			{'c', 0, 1, 4},
+			{'d', 'd', 0, 1},
+			{'c', 0, 1, 7},
+			{'d', 0, 1, 5},
+			{'e', 'f', 0, 2},
+		},
+	},
+}
+
+func lookup(c *contractTrieSet, nnode int, s []uint8) (i, n int) {
+	scan := c.scanner(0, nnode, s)
+	scan.scan(0)
+	return scan.result()
+}
+
+func TestLookupContraction(t *testing.T) {
+	for i, tt := range lookupTests {
+		cts := contractTrieSet(tt.tries)
+		for j, lu := range tt.lookup {
+			str := lu.str
+			for _, s := range []string{str, str + "X"} {
+				const msg = "%d:%d: %s of %q %v; want %v"
+				offset, n := lookup(&cts, tt.n, []byte(s))
+				if offset != lu.offset {
+					t.Errorf(msg, i, j, "offset", s, offset, lu.offset)
+				}
+				if n != lu.n {
+					t.Errorf(msg, i, j, "bytes consumed", s, n, len(str))
+				}
+			}
+		}
+	}
+}

vendor/golang.org/x/text/internal/colltab/iter.go

@@ -0,0 +1,182 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package colltab
+
+import (
+	"golang.org/x/text/collate/colltab"
+)
+
+// An Iter incrementally converts chunks of the input text to collation
+// elements, while ensuring that the collation elements are in normalized order
+// (that is, they are in the order as if the input text were normalized first).
+type Iter struct {
+	Weighter colltab.Weighter
+	Elems    []colltab.Elem
+	// N is the number of elements in Elems that will not be reordered on
+	// subsequent iterations, N <= len(Elems).
+	N int
+
+	bytes []byte
+	str   string
+	// Because the Elems buffer may contain collation elements that are needed
+	// for look-ahead, we need two positions in the text (bytes or str): one for
+	// the end position in the text for the current iteration and one for the
+	// start of the next call to appendNext.
+	pEnd  int // end position in text corresponding to N.
+	pNext int // pEnd <= pNext.
+}
+
+// Reset sets the position in the current input text to p and discards any
+// results obtained so far.
+func (i *Iter) Reset(p int) {
+	i.Elems = i.Elems[:0]
+	i.N = 0
+	i.pEnd = p
+	i.pNext = p
+}
+
+// Len returns the length of the input text.
+func (i *Iter) Len() int {
+	if i.bytes != nil {
+		return len(i.bytes)
+	}
+	return len(i.str)
+}
+
+// Discard removes the collation elements up to N.
+func (i *Iter) Discard() {
+	// TODO: change this such that only modifiers following starters will have
+	// to be copied.
+	i.Elems = i.Elems[:copy(i.Elems, i.Elems[i.N:])]
+	i.N = 0
+}
+
+// End returns the end position of the input text for which Next has returned
+// results.
+func (i *Iter) End() int {
+	return i.pEnd
+}
+
+// SetInput resets i to input s.
+func (i *Iter) SetInput(s []byte) {
+	i.bytes = s
+	i.str = ""
+	i.Reset(0)
+}
+
+// SetInputString resets i to input s.
+func (i *Iter) SetInputString(s string) {
+	i.str = s
+	i.bytes = nil
+	i.Reset(0)
+}
+
+func (i *Iter) done() bool {
+	return i.pNext >= len(i.str) && i.pNext >= len(i.bytes)
+}
+
+func (i *Iter) appendNext() bool {
+	if i.done() {
+		return false
+	}
+	var sz int
+	if i.bytes == nil {
+		i.Elems, sz = i.Weighter.AppendNextString(i.Elems, i.str[i.pNext:])
+	} else {
+		i.Elems, sz = i.Weighter.AppendNext(i.Elems, i.bytes[i.pNext:])
+	}
+	if sz == 0 {
+		sz = 1
+	}
+	i.pNext += sz
+	return true
+}
+
+// Next appends Elems to the internal array. On each iteration, it will either
+// add starters or modifiers. In the majority of cases, an Elem with a primary
+// value > 0 will have a CCC of 0. The CCC values of collation elements are also
+// used to detect if the input string was not normalized and to adjust the
+// result accordingly.
+func (i *Iter) Next() bool {
+	if i.N == len(i.Elems) && !i.appendNext() {
+		return false
+	}
+
+	// Check if the current segment starts with a starter.
+	prevCCC := i.Elems[len(i.Elems)-1].CCC()
+	if prevCCC == 0 {
+		i.N = len(i.Elems)
+		i.pEnd = i.pNext
+		return true
+	} else if i.Elems[i.N].CCC() == 0 {
+		// set i.N to only cover part of i.Elems for which prevCCC == 0 and
+		// use rest for the next call to next.
+		for i.N++; i.N < len(i.Elems) && i.Elems[i.N].CCC() == 0; i.N++ {
+		}
+		i.pEnd = i.pNext
+		return true
+	}
+
+	// The current (partial) segment starts with modifiers. We need to collect
+	// all successive modifiers to ensure that they are normalized.
+	for {
+		p := len(i.Elems)
+		i.pEnd = i.pNext
+		if !i.appendNext() {
+			break
+		}
+
+		if ccc := i.Elems[p].CCC(); ccc == 0 || len(i.Elems)-i.N > maxCombiningCharacters {
+			// Leave the starter for the next iteration. This ensures that we
+			// do not return sequences of collation elements that cross two
+			// segments.
+			//
+			// TODO: handle large number of combining characters by fully
+			// normalizing the input segment before iteration. This ensures
+			// results are consistent across the text repo.
+			i.N = p
+			return true
+		} else if ccc < prevCCC {
+			i.doNorm(p, ccc) // should be rare, never occurs for NFD and FCC.
+		} else {
+			prevCCC = ccc
+		}
+	}
+
+	done := len(i.Elems) != i.N
+	i.N = len(i.Elems)
+	return done
+}
+
+// nextNoNorm is the same as next, but does not "normalize" the collation
+// elements.
+func (i *Iter) nextNoNorm() bool {
+	// TODO: remove this function. Using this instead of next does not seem
+	// to improve performance in any significant way. We retain this until
+	// later for evaluation purposes.
+	if i.done() {
+		return false
+	}
+	i.appendNext()
+	i.N = len(i.Elems)
+	return true
+}
+
+const maxCombiningCharacters = 30
+
+// doNorm reorders the collation elements in i.Elems.
+// It assumes that blocks of collation elements added with appendNext
+// either start and end with the same CCC or start with CCC == 0.
+// This allows for a single insertion point for the entire block.
+// The correctness of this assumption is verified in builder.go.
+func (i *Iter) doNorm(p int, ccc uint8) {
+	n := len(i.Elems)
+	k := p
+	for p--; p > i.N && ccc < i.Elems[p-1].CCC(); p-- {
+	}
+	i.Elems = append(i.Elems, i.Elems[p:k]...)
+	copy(i.Elems[p:], i.Elems[k:])
+	i.Elems = i.Elems[:n]
+}

vendor/golang.org/x/text/internal/colltab/iter_test.go

@@ -0,0 +1,77 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package colltab
+
+import (
+	"testing"
+
+	"golang.org/x/text/collate/colltab"
+)
+
+const (
+	defaultSecondary = 0x20
+)
+
+func makeCE(w []int) colltab.Elem {
+	ce, err := colltab.MakeElem(w[0], w[1], w[2], uint8(w[3]))
+	if err != nil {
+		panic(err)
+	}
+	return ce
+}
+
+func TestDoNorm(t *testing.T) {
+	const div = -1 // The insertion point of the next block.
+	tests := []struct {
+		in, out []int
+	}{{
+		in:  []int{4, div, 3},
+		out: []int{3, 4},
+	}, {
+		in:  []int{4, div, 3, 3, 3},
+		out: []int{3, 3, 3, 4},
+	}, {
+		in:  []int{0, 4, div, 3},
+		out: []int{0, 3, 4},
+	}, {
+		in:  []int{0, 0, 4, 5, div, 3, 3},
+		out: []int{0, 0, 3, 3, 4, 5},
+	}, {
+		in:  []int{0, 0, 1, 4, 5, div, 3, 3},
+		out: []int{0, 0, 1, 3, 3, 4, 5},
+	}, {
+		in:  []int{0, 0, 1, 4, 5, div, 4, 4},
+		out: []int{0, 0, 1, 4, 4, 4, 5},
+	},
+	}
+	for j, tt := range tests {
+		i := Iter{}
+		var w, p int
+		for k, cc := range tt.in {
+
+			if cc == div {
+				w = 100
+				p = k
+				continue
+			}
+			i.Elems = append(i.Elems, makeCE([]int{w, defaultSecondary, 2, cc}))
+		}
+		i.doNorm(p, i.Elems[p].CCC())
+		if len(i.Elems) != len(tt.out) {
+			t.Errorf("%d: length was %d; want %d", j, len(i.Elems), len(tt.out))
+		}
+		prevCCC := uint8(0)
+		for k, ce := range i.Elems {
+			if int(ce.CCC()) != tt.out[k] {
+				t.Errorf("%d:%d: unexpected CCC. Was %d; want %d", j, k, ce.CCC(), tt.out[k])
+			}
+			if k > 0 && ce.CCC() == prevCCC && i.Elems[k-1].Primary() > ce.Primary() {
+				t.Errorf("%d:%d: normalization crossed across CCC boundary.", j, k)
+			}
+		}
+	}
+
+	// Combining rune overflow is tested in search/pattern_test.go.
+}