1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
|
package ucd
import (
"bufio"
"encoding/binary"
"encoding/hex"
"io"
"regexp"
"strings"
)
type CodePointRange struct {
From rune
To rune
}
var codePointRangeNil = &CodePointRange{
From: 0,
To: 0,
}
type field string
func (f field) codePointRange() (*CodePointRange, error) {
var from, to rune
var err error
cp := reCodePointRange.FindStringSubmatch(string(f))
from, err = decodeHexToRune(cp[1])
if err != nil {
return codePointRangeNil, err
}
if cp[2] != "" {
to, err = decodeHexToRune(cp[2])
if err != nil {
return codePointRangeNil, err
}
} else {
to = from
}
return &CodePointRange{
From: from,
To: to,
}, nil
}
func decodeHexToRune(hexCodePoint string) (rune, error) {
h := hexCodePoint
if len(h)%2 != 0 {
h = "0" + h
}
b, err := hex.DecodeString(h)
if err != nil {
return 0, err
}
l := len(b)
for i := 0; i < 4-l; i++ {
b = append([]byte{0}, b...)
}
n := binary.BigEndian.Uint32(b)
return rune(n), nil
}
func (f field) symbol() string {
return string(f)
}
func (f field) normalizedSymbol() string {
return normalizeSymbolicValue(string(f))
}
var symValReplacer = strings.NewReplacer("_", "", "-", "", "\x20", "")
// normalizeSymbolicValue normalizes a symbolic value. The normalized value meets UAX44-LM3.
//
// https://www.unicode.org/reports/tr44/#UAX44-LM3
func normalizeSymbolicValue(s string) string {
v := strings.ToLower(symValReplacer.Replace(s))
if strings.HasPrefix(v, "is") && v != "is" {
return v[2:]
}
return v
}
var (
reLine = regexp.MustCompile(`^\s*(.*?)\s*(#.*)?$`)
reCodePointRange = regexp.MustCompile(`^([[:xdigit:]]+)(?:..([[:xdigit:]]+))?$`)
specialCommentPrefix = "# @missing:"
)
// This parser can parse data files of Unicode Character Database (UCD).
// Specifically, it has the following two functions:
// - Converts each line of the data files into a slice of fields.
// - Recognizes specially-formatted comments starting `@missing` and generates a slice of fields.
//
// However, for practical purposes, each field needs to be analyzed more specifically.
// For instance, in UnicodeData.txt, the first field represents a range of code points,
// so it needs to be recognized as a hexadecimal string.
// You can perform more specific parsing for each file by implementing a dedicated parser that wraps this parser.
//
// https://www.unicode.org/reports/tr44/#Format_Conventions
type parser struct {
scanner *bufio.Scanner
fields []field
defaultFields []field
err error
fieldBuf []field
defaultFieldBuf []field
}
func newParser(r io.Reader) *parser {
return &parser{
scanner: bufio.NewScanner(r),
fieldBuf: make([]field, 50),
defaultFieldBuf: make([]field, 50),
}
}
func (p *parser) parse() bool {
for p.scanner.Scan() {
p.parseRecord(p.scanner.Text())
if p.fields != nil || p.defaultFields != nil {
return true
}
}
p.err = p.scanner.Err()
return false
}
func (p *parser) parseRecord(src string) {
ms := reLine.FindStringSubmatch(src)
mFields := ms[1]
mComment := ms[2]
if mFields != "" {
p.fields = parseFields(p.fieldBuf, mFields)
} else {
p.fields = nil
}
if strings.HasPrefix(mComment, specialCommentPrefix) {
p.defaultFields = parseFields(p.defaultFieldBuf, strings.Replace(mComment, specialCommentPrefix, "", -1))
} else {
p.defaultFields = nil
}
}
func parseFields(buf []field, src string) []field {
n := 0
for _, f := range strings.Split(src, ";") {
buf[n] = field(strings.TrimSpace(f))
n++
}
return buf[:n]
}
|
