Update Go library to r60.

From-SVN: r178910

libgo/go/unicode/casetables.go

@@ -9,7 +9,6 @@
 
 package unicode
 
-
 var TurkishCase = _TurkishCase
 var _TurkishCase = SpecialCase{
 	CaseRange{0x0049, 0x0049, d{0, 0x131 - 0x49, 0}},

libgo/go/unicode/digit.go

@@ -6,7 +6,7 @@ package unicode
 
 // IsDigit reports whether the rune is a decimal digit.
 func IsDigit(rune int) bool {
-	if rune < 0x100 { // quick ASCII (Latin-1, really) check
+	if rune <= MaxLatin1 {
 		return '0' <= rune && rune <= '9'
 	}
 	return Is(Digit, rune)

libgo/go/unicode/digit_test.go

@@ -118,7 +118,7 @@ func TestDigit(t *testing.T) {
 
 // Test that the special case in IsDigit agrees with the table
 func TestDigitOptimization(t *testing.T) {
-	for i := 0; i < 0x100; i++ {
+	for i := 0; i <= MaxLatin1; i++ {
 		if Is(Digit, i) != IsDigit(i) {
 			t.Errorf("IsDigit(U+%04X) disagrees with Is(Digit)", i)
 		}

libgo/go/unicode/graphic.go

@@ -0,0 +1,132 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package unicode
+
+// Bit masks for each code point under U+0100, for fast lookup.
+const (
+	pC  = 1 << iota // a control character.
+	pP              // a punctuation character.
+	pN              // a numeral.
+	pS              // a symbolic character.
+	pZ              // a spacing character.
+	pLu             // an upper-case letter.
+	pLl             // a lower-case letter.
+	pp              // a printable character according to Go's definition.
+	pg  = pp | pZ   // a graphical character according to the Unicode definition.
+)
+
+// GraphicRanges defines the set of graphic characters according to Unicode.
+var GraphicRanges = []*RangeTable{
+	L, M, N, P, S, Zs,
+}
+
+// PrintRanges defines the set of printable characters according to Go.
+// ASCII space, U+0020, is handled separately.
+var PrintRanges = []*RangeTable{
+	L, M, N, P, S,
+}
+
+// IsGraphic reports whether the rune is defined as a Graphic by Unicode.
+// Such characters include letters, marks, numbers, punctuation, symbols, and
+// spaces, from categories L, M, N, P, S, Zs.
+func IsGraphic(rune int) bool {
+	// We cast to uint32 to avoid the extra test for negative,
+	// and in the index we cast to uint8 to avoid the range check.
+	if uint32(rune) <= MaxLatin1 {
+		return properties[uint8(rune)]&pg != 0
+	}
+	return IsOneOf(GraphicRanges, rune)
+}
+
+// IsPrint reports whether the rune is defined as printable by Go. Such
+// characters include letters, marks, numbers, punctuation, symbols, and the
+// ASCII space character, from categories L, M, N, P, S and the ASCII space
+// character.  This categorization is the same as IsGraphic except that the
+// only spacing character is ASCII space, U+0020.
+func IsPrint(rune int) bool {
+	if uint32(rune) <= MaxLatin1 {
+		return properties[uint8(rune)]&pp != 0
+	}
+	return IsOneOf(PrintRanges, rune)
+}
+
+// IsOneOf reports whether the rune is a member of one of the ranges.
+// The rune is known to be above Latin-1.
+func IsOneOf(set []*RangeTable, rune int) bool {
+	for _, inside := range set {
+		if Is(inside, rune) {
+			return true
+		}
+	}
+	return false
+}
+
+// IsControl reports whether the rune is a control character.
+// The C (Other) Unicode category includes more code points
+// such as surrogates; use Is(C, rune) to test for them.
+func IsControl(rune int) bool {
+	if uint32(rune) <= MaxLatin1 {
+		return properties[uint8(rune)]&pC != 0
+	}
+	// All control characters are < Latin1Max.
+	return false
+}
+
+// IsLetter reports whether the rune is a letter (category L).
+func IsLetter(rune int) bool {
+	if uint32(rune) <= MaxLatin1 {
+		return properties[uint8(rune)]&(pLu|pLl) != 0
+	}
+	return Is(Letter, rune)
+}
+
+// IsMark reports whether the rune is a mark character (category M).
+func IsMark(rune int) bool {
+	// There are no mark characters in Latin-1.
+	return Is(Mark, rune)
+}
+
+// IsNumber reports whether the rune is a number (category N).
+func IsNumber(rune int) bool {
+	if uint32(rune) <= MaxLatin1 {
+		return properties[uint8(rune)]&pN != 0
+	}
+	return Is(Number, rune)
+}
+
+// IsPunct reports whether the rune is a Unicode punctuation character
+// (category P).
+func IsPunct(rune int) bool {
+	if uint32(rune) <= MaxLatin1 {
+		return properties[uint8(rune)]&pP != 0
+	}
+	return Is(Punct, rune)
+}
+
+// IsSpace reports whether the rune is a space character as defined
+// by Unicode's White Space property; in the Latin-1 space
+// this is
+//	'\t', '\n', '\v', '\f', '\r', ' ', U+0085 (NEL), U+00A0 (NBSP).
+// Other definitions of spacing characters are set by category
+// Z and property Pattern_White_Space.
+func IsSpace(rune int) bool {
+	// This property isn't the same as Z; special-case it.
+	if uint32(rune) <= MaxLatin1 {
+		switch rune {
+		case '\t', '\n', '\v', '\f', '\r', ' ', 0x85, 0xA0:
+			return true
+		}
+		return false
+	}
+	return Is(White_Space, rune)
+}
+
+// IsSymbol reports whether the rune is a symbolic character.
+func IsSymbol(rune int) bool {
+	if uint32(rune) <= MaxLatin1 {
+		return properties[uint8(rune)]&pS != 0
+	}
+	return Is(Symbol, rune)
+}

libgo/go/unicode/graphic_test.go

@@ -0,0 +1,122 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package unicode_test
+
+import (
+	"testing"
+	. "unicode"
+)
+
+// Independently check that the special "Is" functions work
+// in the Latin-1 range through the property table.
+
+func TestIsControlLatin1(t *testing.T) {
+	for i := 0; i <= MaxLatin1; i++ {
+		got := IsControl(i)
+		want := false
+		switch {
+		case 0x00 <= i && i <= 0x1F:
+			want = true
+		case 0x7F <= i && i <= 0x9F:
+			want = true
+		}
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}
+
+func TestIsLetterLatin1(t *testing.T) {
+	for i := 0; i <= MaxLatin1; i++ {
+		got := IsLetter(i)
+		want := Is(Letter, i)
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}
+
+func TestIsUpperLatin1(t *testing.T) {
+	for i := 0; i <= MaxLatin1; i++ {
+		got := IsUpper(i)
+		want := Is(Upper, i)
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}
+
+func TestIsLowerLatin1(t *testing.T) {
+	for i := 0; i <= MaxLatin1; i++ {
+		got := IsLower(i)
+		want := Is(Lower, i)
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}
+
+func TestNumberLatin1(t *testing.T) {
+	for i := 0; i <= MaxLatin1; i++ {
+		got := IsNumber(i)
+		want := Is(Number, i)
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}
+
+func TestIsPrintLatin1(t *testing.T) {
+	for i := 0; i <= MaxLatin1; i++ {
+		got := IsPrint(i)
+		want := IsOneOf(PrintRanges, i)
+		if i == ' ' {
+			want = true
+		}
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}
+
+func TestIsGraphicLatin1(t *testing.T) {
+	for i := 0; i <= MaxLatin1; i++ {
+		got := IsGraphic(i)
+		want := IsOneOf(GraphicRanges, i)
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}
+
+func TestIsPunctLatin1(t *testing.T) {
+	for i := 0; i <= MaxLatin1; i++ {
+		got := IsPunct(i)
+		want := Is(Punct, i)
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}
+
+func TestIsSpaceLatin1(t *testing.T) {
+	for i := 0; i <= MaxLatin1; i++ {
+		got := IsSpace(i)
+		want := Is(White_Space, i)
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}
+
+func TestIsSymbolLatin1(t *testing.T) {
+	for i := 0; i <= MaxLatin1; i++ {
+		got := IsSymbol(i)
+		want := Is(Symbol, i)
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}

libgo/go/unicode/letter.go

@@ -9,15 +9,35 @@ package unicode
 const (
 	MaxRune         = 0x10FFFF // Maximum valid Unicode code point.
 	ReplacementChar = 0xFFFD   // Represents invalid code points.
+	MaxASCII        = 0x7F     // maximum ASCII value.
+	MaxLatin1       = 0xFF     // maximum Latin-1 value.
 )
 
+// RangeTable defines a set of Unicode code points by listing the ranges of
+// code points within the set. The ranges are listed in two slices
+// to save space: a slice of 16-bit ranges and a slice of 32-bit ranges.
+// The two slices must be in sorted order and non-overlapping.
+// Also, R32 should contain only values >= 0x10000 (1<<16).
+type RangeTable struct {
+	R16 []Range16
+	R32 []Range32
+}
 
-// The representation of a range of Unicode code points.  The range runs from Lo to Hi
+// Range16 represents of a range of 16-bit Unicode code points.  The range runs from Lo to Hi
 // inclusive and has the specified stride.
-type Range struct {
-	Lo     int
-	Hi     int
-	Stride int
+type Range16 struct {
+	Lo     uint16
+	Hi     uint16
+	Stride uint16
+}
+
+// Range32 represents of a range of Unicode code points and is used when one or
+// more of the values will not fit in 16 bits.  The range runs from Lo to Hi
+// inclusive and has the specified stride. Lo and Hi must always be >= 1<<16.
+type Range32 struct {
+	Lo     uint32
+	Hi     uint32
+	Stride uint32
 }
 
 // CaseRange represents a range of Unicode code points for simple (one
@@ -31,8 +51,8 @@ type Range struct {
 //	{UpperLower, UpperLower, UpperLower}
 // The constant UpperLower has an otherwise impossible delta value.
 type CaseRange struct {
-	Lo    int
-	Hi    int
+	Lo    uint32
+	Hi    uint32
 	Delta d
 }
 
@@ -60,22 +80,8 @@ const (
 	UpperLower = MaxRune + 1 // (Cannot be a valid delta.)
 )
 
-// Is tests whether rune is in the specified table of ranges.
-func Is(ranges []Range, rune int) bool {
-	// common case: rune is ASCII or Latin-1
-	if rune < 0x100 {
-		for _, r := range ranges {
-			if rune > r.Hi {
-				continue
-			}
-			if rune < r.Lo {
-				return false
-			}
-			return (rune-r.Lo)%r.Stride == 0
-		}
-		return false
-	}
-
+// is16 uses binary search to test whether rune is in the specified slice of 16-bit ranges.
+func is16(ranges []Range16, rune uint16) bool {
 	// binary search over ranges
 	lo := 0
 	hi := len(ranges)
@@ -94,51 +100,80 @@ func Is(ranges []Range, rune int) bool {
 	return false
 }
 
+// is32 uses binary search to test whether rune is in the specified slice of 32-bit ranges.
+func is32(ranges []Range32, rune uint32) bool {
+	// binary search over ranges
+	lo := 0
+	hi := len(ranges)
+	for lo < hi {
+		m := lo + (hi-lo)/2
+		r := ranges[m]
+		if r.Lo <= rune && rune <= r.Hi {
+			return (rune-r.Lo)%r.Stride == 0
+		}
+		if rune < r.Lo {
+			hi = m
+		} else {
+			lo = m + 1
+		}
+	}
+	return false
+}
+
+// Is tests whether rune is in the specified table of ranges.
+func Is(rangeTab *RangeTable, rune int) bool {
+	// common case: rune is ASCII or Latin-1.
+	if uint32(rune) <= MaxLatin1 {
+		// Only need to check R16, since R32 is always >= 1<<16.
+		r16 := uint16(rune)
+		for _, r := range rangeTab.R16 {
+			if r16 > r.Hi {
+				continue
+			}
+			if r16 < r.Lo {
+				return false
+			}
+			return (r16-r.Lo)%r.Stride == 0
+		}
+		return false
+	}
+	r16 := rangeTab.R16
+	if len(r16) > 0 && rune <= int(r16[len(r16)-1].Hi) {
+		return is16(r16, uint16(rune))
+	}
+	r32 := rangeTab.R32
+	if len(r32) > 0 && rune >= int(r32[0].Lo) {
+		return is32(r32, uint32(rune))
+	}
+	return false
+}
+
 // IsUpper reports whether the rune is an upper case letter.
 func IsUpper(rune int) bool {
-	if rune < 0x80 { // quick ASCII check
-		return 'A' <= rune && rune <= 'Z'
+	// See comment in IsGraphic.
+	if uint32(rune) <= MaxLatin1 {
+		return properties[uint8(rune)]&pLu != 0
 	}
 	return Is(Upper, rune)
 }
 
 // IsLower reports whether the rune is a lower case letter.
 func IsLower(rune int) bool {
-	if rune < 0x80 { // quick ASCII check
-		return 'a' <= rune && rune <= 'z'
+	// See comment in IsGraphic.
+	if uint32(rune) <= MaxLatin1 {
+		return properties[uint8(rune)]&pLl != 0
 	}
 	return Is(Lower, rune)
 }
 
 // IsTitle reports whether the rune is a title case letter.
 func IsTitle(rune int) bool {
-	if rune < 0x80 { // quick ASCII check
+	if rune <= MaxLatin1 {
 		return false
 	}
 	return Is(Title, rune)
 }
 
-// IsLetter reports whether the rune is a letter.
-func IsLetter(rune int) bool {
-	if rune < 0x80 { // quick ASCII check
-		rune &^= 'a' - 'A'
-		return 'A' <= rune && rune <= 'Z'
-	}
-	return Is(Letter, rune)
-}
-
-// IsSpace reports whether the rune is a white space character.
-func IsSpace(rune int) bool {
-	if rune <= 0xFF { // quick Latin-1 check
-		switch rune {
-		case '\t', '\n', '\v', '\f', '\r', ' ', 0x85, 0xA0:
-			return true
-		}
-		return false
-	}
-	return Is(White_Space, rune)
-}
-
 // to maps the rune using the specified case mapping.
 func to(_case int, rune int, caseRange []CaseRange) int {
 	if _case < 0 || MaxCase <= _case {
@@ -150,7 +185,7 @@ func to(_case int, rune int, caseRange []CaseRange) int {
 	for lo < hi {
 		m := lo + (hi-lo)/2
 		r := caseRange[m]
-		if r.Lo <= rune && rune <= r.Hi {
+		if int(r.Lo) <= rune && rune <= int(r.Hi) {
 			delta := int(r.Delta[_case])
 			if delta > MaxRune {
 				// In an Upper-Lower sequence, which always starts with
@@ -163,11 +198,11 @@ func to(_case int, rune int, caseRange []CaseRange) int {
 				// bit in the sequence offset.
 				// The constants UpperCase and TitleCase are even while LowerCase
 				// is odd so we take the low bit from _case.
-				return r.Lo + ((rune-r.Lo)&^1 | _case&1)
+				return int(r.Lo) + ((rune-int(r.Lo))&^1 | _case&1)
 			}
 			return rune + delta
 		}
-		if rune < r.Lo {
+		if rune < int(r.Lo) {
 			hi = m
 		} else {
 			lo = m + 1
@@ -183,7 +218,7 @@ func To(_case int, rune int) int {
 
 // ToUpper maps the rune to upper case.
 func ToUpper(rune int) int {
-	if rune < 0x80 { // quick ASCII check
+	if rune <= MaxASCII {
 		if 'a' <= rune && rune <= 'z' {
 			rune -= 'a' - 'A'
 		}
@@ -194,7 +229,7 @@ func ToUpper(rune int) int {
 
 // ToLower maps the rune to lower case.
 func ToLower(rune int) int {
-	if rune < 0x80 { // quick ASCII check
+	if rune <= MaxASCII {
 		if 'A' <= rune && rune <= 'Z' {
 			rune += 'a' - 'A'
 		}
@@ -205,7 +240,7 @@ func ToLower(rune int) int {
 
 // ToTitle maps the rune to title case.
 func ToTitle(rune int) int {
-	if rune < 0x80 { // quick ASCII check
+	if rune <= MaxASCII {
 		if 'a' <= rune && rune <= 'z' { // title case is upper case for ASCII
 			rune -= 'a' - 'A'
 		}
@@ -240,3 +275,52 @@ func (special SpecialCase) ToLower(rune int) int {
 	}
 	return r
 }
+
+// caseOrbit is defined in tables.go as []foldPair.  Right now all the
+// entries fit in uint16, so use uint16.  If that changes, compilation
+// will fail (the constants in the composite literal will not fit in uint16)
+// and the types here can change to uint32.
+type foldPair struct {
+	From uint16
+	To   uint16
+}
+
+// SimpleFold iterates over Unicode code points equivalent under
+// the Unicode-defined simple case folding.  Among the code points
+// equivalent to rune (including rune itself), SimpleFold returns the
+// smallest r >= rune if one exists, or else the smallest r >= 0. 
+//
+// For example:
+//	SimpleFold('A') = 'a'
+//	SimpleFold('a') = 'A'
+//
+//	SimpleFold('K') = 'k'
+//	SimpleFold('k') = '\u212A' (Kelvin symbol, K)
+//	SimpleFold('\u212A') = 'K'
+//
+//	SimpleFold('1') = '1'
+//
+func SimpleFold(rune int) int {
+	// Consult caseOrbit table for special cases.
+	lo := 0
+	hi := len(caseOrbit)
+	for lo < hi {
+		m := lo + (hi-lo)/2
+		if int(caseOrbit[m].From) < rune {
+			lo = m + 1
+		} else {
+			hi = m
+		}
+	}
+	if lo < len(caseOrbit) && int(caseOrbit[lo].From) == rune {
+		return int(caseOrbit[lo].To)
+	}
+
+	// No folding specified.  This is a one- or two-element
+	// equivalence class containing rune and ToLower(rune)
+	// and ToUpper(rune) if they are different from rune.
+	if l := ToLower(rune); l != rune {
+		return l
+	}
+	return ToUpper(rune)
+}

libgo/go/unicode/letter_test.go

@@ -212,6 +212,10 @@ var caseTest = []caseT{
 	{UpperCase, 0x10450, 0x10450},
 	{LowerCase, 0x10450, 0x10450},
 	{TitleCase, 0x10450, 0x10450},
+
+	// Non-letters with case.
+	{LowerCase, 0x2161, 0x2171},
+	{UpperCase, 0x0345, 0x0399},
 }
 
 func TestIsLetter(t *testing.T) {
@@ -323,7 +327,7 @@ func TestIsSpace(t *testing.T) {
 // Check that the optimizations for IsLetter etc. agree with the tables.
 // We only need to check the Latin-1 range.
 func TestLetterOptimizations(t *testing.T) {
-	for i := 0; i < 0x100; i++ {
+	for i := 0; i <= MaxLatin1; i++ {
 		if Is(Letter, i) != IsLetter(i) {
 			t.Errorf("IsLetter(U+%04X) disagrees with Is(Letter)", i)
 		}
@@ -376,3 +380,49 @@ func TestTurkishCase(t *testing.T) {
 		}
 	}
 }
+
+var simpleFoldTests = []string{
+	// SimpleFold could order its returned slices in any order it wants,
+	// but we know it orders them in increasing order starting at in
+	// and looping around from MaxRune to 0.
+
+	// Easy cases.
+	"Aa",
+	"aA",
+	"δΔ",
+	"Δδ",
+
+	// ASCII special cases.
+	"KkK",
+	"kKK",
+	"KKk",
+	"Ssſ",
+	"sſS",
+	"ſSs",
+
+	// Non-ASCII special cases.
+	"ρϱΡ",
+	"ϱΡρ",
+	"Ρρϱ",
+	"ͅΙιι",
+	"Ιιιͅ",
+	"ιιͅΙ",
+	"ιͅΙι",
+
+	// Extra special cases: has lower/upper but no case fold.
+	"İ",
+	"ı",
+}
+
+func TestSimpleFold(t *testing.T) {
+	for _, tt := range simpleFoldTests {
+		cycle := []int(tt)
+		rune := cycle[len(cycle)-1]
+		for _, out := range cycle {
+			if r := SimpleFold(rune); r != out {
+				t.Errorf("SimpleFold(%#U) = %#U, want %#U", rune, r, out)
+			}
+			rune = out
+		}
+	}
+}

libgo/go/unicode/script_test.go

@@ -149,7 +149,14 @@ var inCategoryTest = []T{
 	{0x2028, "Zl"},
 	{0x2029, "Zp"},
 	{0x202f, "Zs"},
-	{0x04aa, "letter"},
+	// Unifieds.
+	{0x04aa, "L"},
+	{0x0009, "C"},
+	{0x1712, "M"},
+	{0x0031, "N"},
+	{0x00bb, "P"},
+	{0x00a2, "S"},
+	{0x00a0, "Z"},
 }
 
 var inPropTest = []T{
@@ -197,13 +204,13 @@ func TestScripts(t *testing.T) {
 			t.Fatal(test.script, "not a known script")
 		}
 		if !Is(Scripts[test.script], test.rune) {
-			t.Errorf("IsScript(%#x, %s) = false, want true", test.rune, test.script)
+			t.Errorf("IsScript(%U, %s) = false, want true", test.rune, test.script)
 		}
 		notTested[test.script] = false, false
 	}
 	for _, test := range outTest {
 		if Is(Scripts[test.script], test.rune) {
-			t.Errorf("IsScript(%#x, %s) = true, want false", test.rune, test.script)
+			t.Errorf("IsScript(%U, %s) = true, want false", test.rune, test.script)
 		}
 	}
 	for k := range notTested {
@@ -221,7 +228,7 @@ func TestCategories(t *testing.T) {
 			t.Fatal(test.script, "not a known category")
 		}
 		if !Is(Categories[test.script], test.rune) {
-			t.Errorf("IsCategory(%#x, %s) = false, want true", test.rune, test.script)
+			t.Errorf("IsCategory(%U, %s) = false, want true", test.rune, test.script)
 		}
 		notTested[test.script] = false, false
 	}
@@ -240,7 +247,7 @@ func TestProperties(t *testing.T) {
 			t.Fatal(test.script, "not a known prop")
 		}
 		if !Is(Properties[test.script], test.rune) {
-			t.Errorf("IsCategory(%#x, %s) = false, want true", test.rune, test.script)
+			t.Errorf("IsCategory(%U, %s) = false, want true", test.rune, test.script)
 		}
 		notTested[test.script] = false, false
 	}